예제 #1
0
        public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
                                ref IDocumentData[] documents, ProcessInfo processInfo)
        {
            int  ind        = param.GetParam <int>("Column").Value;
            bool descending = param.GetParam <bool>("Descending").Value;

            if (ind < mdata.ColumnCount)
            {
                BaseVector v = mdata.Values.GetColumn(ind);
                int[]      o = ArrayUtils.Order(v);
                if (descending)
                {
                    ArrayUtils.Revert(o);
                }
                mdata.ExtractRows(o);
            }
            else
            {
                double[] v = mdata.NumericColumns[ind - mdata.ColumnCount];
                int[]    o = ArrayUtils.Order(v);
                if (descending)
                {
                    ArrayUtils.Revert(o);
                }
                mdata.ExtractRows(o);
            }
        }
예제 #2
0
 internal static IMatrixData NonzeroFilter1Split(bool rows, int minValids, bool percentage, IMatrixData mdata,
                                                 Parameters param, double threshold, double threshold2, FilteringMode filterMode)
 {
     if (rows)
     {
         IMatrixData supplTab  = (IMatrixData)mdata.Clone();
         List <int>  valids    = new List <int>();
         List <int>  notvalids = new List <int>();
         for (int i = 0; i < mdata.RowCount; i++)
         {
             int count = 0;
             for (int j = 0; j < mdata.ColumnCount; j++)
             {
                 if ((IsValid(mdata.Values.Get(i, j), threshold, threshold2, filterMode)))
                 {
                     count++;
                 }
             }
             if ((Valid(count, minValids, percentage, mdata.ColumnCount)))
             {
                 valids.Add(i);
             }
             else
             {
                 notvalids.Add(i);
             }
         }
         //  FilterRowsNew(mdata, param, valids.ToArray());
         supplTab.ExtractRows(notvalids.ToArray());
         return(supplTab);
     }
     else
     {
         IMatrixData supplTab  = (IMatrixData)mdata.Clone();
         List <int>  valids    = new List <int>();
         List <int>  notvalids = new List <int>();
         for (int j = 0; j < mdata.ColumnCount; j++)
         {
             int count = 0;
             for (int i = 0; i < mdata.RowCount; i++)
             {
                 if (IsValid(mdata.Values.Get(i, j), threshold, threshold2, filterMode))
                 {
                     count++;
                 }
             }
             if (Valid(count, minValids, percentage, mdata.RowCount))
             {
                 valids.Add(j);
             }
             else
             {
                 notvalids.Add(j);
             }
         }
         supplTab.ExtractColumns(notvalids.ToArray());
         // FilterColumnsNew(mdata, param, valids.ToArray());
         return(supplTab);
     }
 }
예제 #3
0
        public static IMatrixData CreateSupplTabSplitValids(IMatrixData mdata, int[] rows)
        {
            IMatrixData supplTab = (IMatrixData)mdata.Clone();

            supplTab.ExtractRows(rows);
            return(supplTab);
        }
        public void ModeAtLeastOneGroup(IMatrixData mdata, Dictionary <string, List <string> > samples,
                                        double minValid)
        {
            List <int> validRows = new List <int>();

            for (int i = 0; i < mdata.RowCount; i++)
            {
                bool validNum = false;
                for (int j = 0; j < mdata.ColumnCount; j++)
                {
                    foreach (KeyValuePair <string, List <string> > entry in samples)
                    {
                        if (entry.Value.Contains(mdata.ColumnNames[j]))
                        {
                            if (mdata.Values.Get(i, j) >= minValid)
                            {
                                validNum = true;
                            }
                        }
                    }
                }
                if (validNum)
                {
                    validRows.Add(i);
                }
            }
            mdata.ExtractRows(validRows.ToArray());
        }
        public static void UniqueRows(this IMatrixData mdata, string[] ids, Func <double[], double> combineNumeric,
                                      Func <string[], string> combineString, Func <string[][], string[]> combineCategory,
                                      Func <double[][], double[]> combineMultiNumeric)
        {
            int[]      order          = ArrayUtils.Order(ids);
            List <int> uniqueIdx      = new List <int>();
            string     lastId         = "";
            List <int> idxsWithSameId = new List <int>();

            foreach (int j in order)
            {
                string id = ids[j];
                if (id == lastId)
                {
                    idxsWithSameId.Add(j);
                }
                else
                {
                    CombineRows(mdata, idxsWithSameId, combineNumeric, combineString, combineCategory, combineMultiNumeric);
                    uniqueIdx.Add(j);
                    idxsWithSameId.Clear();
                    idxsWithSameId.Add(j);
                }
                lastId = id;
            }
            CombineRows(mdata, idxsWithSameId, combineNumeric, combineString, combineCategory, combineMultiNumeric);
            mdata.ExtractRows(uniqueIdx.ToArray());
        }
        public void ModeAllGroup(IMatrixData mdata, Dictionary <string, List <string> > samples,
                                 double minValid)
        {
            List <int> validRows = new List <int>();

            for (int i = 0; i < mdata.RowCount; i++)
            {
                Dictionary <string, int> detectNum = new Dictionary <string, int>();
                for (int j = 0; j < mdata.ColumnCount; j++)
                {
                    string preKey   = "";
                    bool   validNum = false;
                    foreach (KeyValuePair <string, List <string> > entry in samples)
                    {
                        if (!detectNum.ContainsKey(entry.Key))
                        {
                            detectNum.Add(entry.Key, 0);
                        }
                        if (entry.Value.Contains(mdata.ColumnNames[j]))
                        {
                            if (preKey == "")
                            {
                                preKey = entry.Key;
                            }
                            else if (preKey != "" && preKey != entry.Key)
                            {
                                if (validNum)
                                {
                                    detectNum[entry.Key]++;
                                    validNum = false;
                                }
                            }
                            if (mdata.Values.Get(i, j) >= minValid)
                            {
                                validNum = true;
                            }
                            preKey = entry.Key;
                        }
                    }
                    if (validNum)
                    {
                        detectNum[preKey]++;
                    }
                }
                bool fail = false;
                foreach (KeyValuePair <string, int> entry in detectNum)
                {
                    if (entry.Value == 0)
                    {
                        fail = true;
                    }
                }
                if (!fail)
                {
                    validRows.Add(i);
                }
            }
            mdata.ExtractRows(validRows.ToArray());
        }
예제 #7
0
        public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
                                ref IDocumentData[] documents, ProcessInfo processInfo)
        {
            var numberOfRows = param.GetParam <int>("Number of rows").Value;
            var indices      = Enumerable.Range(0, numberOfRows).ToArray();

            mdata.ExtractRows(indices);
        }
예제 #8
0
        public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
                                ref IDocumentData[] documents, ProcessInfo processInfo)
        {
            int lines = param.GetParam <int>("Number of rows").Value;

            int[] remains = Enumerable.Range(0, lines).ToArray();
            mdata.ExtractRows(remains);
        }
예제 #9
0
 private static void SortByValues <T>(IMatrixData mdata, T[] v, bool descending) where T : IComparable <T>
 {
     int[] o = ArrayUtils.Order(v);
     if (descending)
     {
         ArrayUtils.Revert(o);
     }
     mdata.ExtractRows(o);
 }
예제 #10
0
        private void RemoveSelectedRowsButton_OnClick(object sender, EventArgs e)
        {
            int[] sel = tableView1.GetSelectedRows();
            if (sel.Length == 0)
            {
                MessageBox.Show(Loc.PleaseSelectSomeRows);
            }
            IMatrixData mx = (IMatrixData)mdata.Clone();

            mx.ExtractRows(ArrayUtils.Complement(sel, tableView1.RowCount));
            createNewMatrix(mx);
        }
        private void KeepSelectedRowsButton_OnClick(object sender, RoutedEventArgs e)
        {
            int[] sel = TableView.GetSelectedRows();
            if (sel.Length == 0)
            {
                MessageBox.Show("Please select some rows.");
            }
            IMatrixData mx = (IMatrixData)mdata.Clone();

            mx.ExtractRows(sel);
            createNewMatrix(mx);
        }
예제 #12
0
        private void KeepSelectedRowsButton_OnClick(object sender, EventArgs e)
        {
            int[] sel = tableView1.GetSelectedRows();
            if (sel.Length == 0)
            {
                MessageBox.Show(Loc.PleaseSelectSomeRows);
            }
            IMatrixData mx = (IMatrixData)mdata.Clone();

            mx.ExtractRows(sel);
            createNewMatrix(mx);
        }
        private void RemoveSelectedRowsButton_OnClick(object sender, RoutedEventArgs e)
        {
            int[] sel = TableView.GetSelectedRows();
            if (sel.Length == 0)
            {
                MessageBox.Show("Please select some rows.");
            }
            IMatrixData mx = (IMatrixData)mdata.Clone();

            mx.ExtractRows(ArrayUtils.Complement(sel, TableView.RowCount));
            createNewMatrix(mx);
        }
예제 #14
0
        public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
                                ref IDocumentData[] documents, ProcessInfo processInfo)
        {
            int nrows = param.GetParam <int>("Number of rows").Value;

            if (nrows >= mdata.RowCount)
            {
                return;
            }
            Random2 rand = new Random2(7);

            int[] inds = rand.NextPermutation(mdata.RowCount).SubArray(nrows);
            mdata.ExtractRows(inds);
        }
예제 #15
0
        public static void FilterRowsNew(IMatrixData mdata, Parameters parameters, int[] rows)
        {
            bool reduceMatrix = UnpackFilterModeParam(parameters) == FilterMode.Reduce;

            if (parameters.GetParam <int>("Filter mode").Value == 0)
            {
                mdata.ExtractRows(rows);
            }
            else if (parameters.GetParam <int>("Filter mode").Value == 1)
            {
                Array.Sort(rows);
                string[][] col = new string[mdata.RowCount][];
                for (int i = 0; i < col.Length; i++)
                {
                    bool contains = Array.BinarySearch(rows, i) >= 0;
                    col[i] = contains ? new[] { "Keep" } : new[] { "Discard" };
                }
                mdata.AddCategoryColumn("Filter", "", col);
            }
            else if (parameters.GetParam <int>("Filter mode").Value == 2)
            {
                mdata.ExtractRows(rows);
            }
        }
        public static void FilterRows(IMatrixData mdata, Parameters parameters, int[] rows)
        {
            bool reduceMatrix = GetReduceMatrix(parameters);

            if (reduceMatrix)
            {
                mdata.ExtractRows(rows);
            }
            else
            {
                Array.Sort(rows);
                string[][] col = new string[mdata.RowCount][];
                for (int i = 0; i < col.Length; i++)
                {
                    bool contains = Array.BinarySearch(rows, i) >= 0;
                    col[i] = contains ? new[] { "Keep" } : new[] { "Discard" };
                }
                mdata.AddCategoryColumn("Filter", "", col);
            }
        }
        public void RemoveLowValuesInRow(IMatrixData mdata, double[] minValids)
        {
            List <int> validRows = new List <int>();

            for (int i = 0; i < mdata.RowCount; i++)
            {
                bool validNum = false;
                for (int j = 0; j < mdata.ColumnCount; j++)
                {
                    if (mdata.Values.Get(i, j) >= minValids[j])
                    {
                        validNum = true;
                    }
                }
                if (validNum)
                {
                    validRows.Add(i);
                }
            }
            mdata.ExtractRows(validRows.ToArray());
        }
 public static void FilterRows(IMatrixData mdata, Parameters parameters, int[] rows)
 {
     bool reduceMatrix = GetReduceMatrix(parameters);
     if (reduceMatrix){
         mdata.ExtractRows(rows);
     } else{
         Array.Sort(rows);
         string[][] col = new string[mdata.RowCount][];
         for (int i = 0; i < col.Length; i++){
             bool contains = Array.BinarySearch(rows, i) >= 0;
             col[i] = contains ? new[]{"Keep"} : new[]{"Discard"};
         }
         mdata.AddCategoryColumn("Filter", "", col);
     }
 }
예제 #19
0
        public void ExtractValues(IMatrixData mdata, Dictionary <string, List <string> > samples,
                                  ParameterWithSubParams <int> minValidType, Dictionary <string, int> minValidAmount, int type)
        {
            int        minValid   = -1;
            List <int> totalCount = new List <int>();

            if (minValidType.Value == 0)
            {
                minValid = minValidType.GetSubParameters().GetParam <int>("Min. raw counts").Value;
            }
            else
            {
                minValid = minValidType.GetSubParameters().GetParam <int>("Min. cpm").Value;
                for (int j = 0; j < mdata.ColumnCount; j++)
                {
                    int Count = 0;
                    foreach (int c in mdata.Values.GetColumn(j))
                    {
                        Count = Count + c;
                    }
                    totalCount.Add(Count);
                }
            }
            List <int> validRows = new List <int>();
            Dictionary <string, int> validNum = new Dictionary <string, int>();

            foreach (KeyValuePair <string, List <string> > entry in samples)
            {
                validNum.Add(entry.Key, 0);
            }
            for (int i = 0; i < mdata.RowCount; i++)
            {
                int totalValids = 0;
                foreach (KeyValuePair <string, List <string> > entry in samples)
                {
                    validNum[entry.Key] = 0;
                }
                for (int j = 0; j < mdata.ColumnCount; j++)
                {
                    foreach (KeyValuePair <string, List <string> > entry in samples)
                    {
                        if (entry.Value.Contains(mdata.ColumnNames[j]))
                        {
                            if (minValidType.Value == 0)
                            {
                                if (mdata.Values.Get(i, j) >= minValid)
                                {
                                    validNum[entry.Key]++;
                                }
                            }
                            else
                            {
                                double cpm = (mdata.Values.Get(i, j) / totalCount[j]) * 1000000;
                                if (cpm >= minValid)
                                {
                                    validNum[entry.Key]++;
                                }
                            }
                        }
                    }
                }
                foreach (KeyValuePair <string, int> entry in validNum)
                {
                    if (validNum[entry.Key] >= minValidAmount[entry.Key])
                    {
                        totalValids++;
                    }
                }
                if ((type == 0) && (totalValids > 0))
                {
                    validRows.Add(i);
                }
                else if ((type == 1) && (totalValids == samples.Count))
                {
                    validRows.Add(i);
                }
                else if (type == 2)
                {
                    int detectSample = 0;
                    foreach (KeyValuePair <string, int> entry in validNum)
                    {
                        detectSample = detectSample + validNum[entry.Key];
                    }
                    if (detectSample >= minValidAmount["total"])
                    {
                        validRows.Add(i);
                    }
                }
            }
            mdata.ExtractRows(validRows.ToArray());
        }