public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int ind = param.GetParam <int>("Column").Value;
bool descending = param.GetParam <bool>("Descending").Value;
if (ind < mdata.ColumnCount)
{
BaseVector v = mdata.Values.GetColumn(ind);
int[] o = ArrayUtils.Order(v);
if (descending)
{
ArrayUtils.Revert(o);
}
mdata.ExtractRows(o);
}
else
{
double[] v = mdata.NumericColumns[ind - mdata.ColumnCount];
int[] o = ArrayUtils.Order(v);
if (descending)
{
ArrayUtils.Revert(o);
}
mdata.ExtractRows(o);
}
}
internal static IMatrixData NonzeroFilter1Split(bool rows, int minValids, bool percentage, IMatrixData mdata,
Parameters param, double threshold, double threshold2, FilteringMode filterMode)
{
if (rows)
{
IMatrixData supplTab = (IMatrixData)mdata.Clone();
List <int> valids = new List <int>();
List <int> notvalids = new List <int>();
for (int i = 0; i < mdata.RowCount; i++)
{
int count = 0;
for (int j = 0; j < mdata.ColumnCount; j++)
{
if ((IsValid(mdata.Values.Get(i, j), threshold, threshold2, filterMode)))
{
count++;
}
}
if ((Valid(count, minValids, percentage, mdata.ColumnCount)))
{
valids.Add(i);
}
else
{
notvalids.Add(i);
}
}
// FilterRowsNew(mdata, param, valids.ToArray());
supplTab.ExtractRows(notvalids.ToArray());
return(supplTab);
}
else
{
IMatrixData supplTab = (IMatrixData)mdata.Clone();
List <int> valids = new List <int>();
List <int> notvalids = new List <int>();
for (int j = 0; j < mdata.ColumnCount; j++)
{
int count = 0;
for (int i = 0; i < mdata.RowCount; i++)
{
if (IsValid(mdata.Values.Get(i, j), threshold, threshold2, filterMode))
{
count++;
}
}
if (Valid(count, minValids, percentage, mdata.RowCount))
{
valids.Add(j);
}
else
{
notvalids.Add(j);
}
}
supplTab.ExtractColumns(notvalids.ToArray());
// FilterColumnsNew(mdata, param, valids.ToArray());
return(supplTab);
}
}
public static IMatrixData CreateSupplTabSplitValids(IMatrixData mdata, int[] rows)
{
IMatrixData supplTab = (IMatrixData)mdata.Clone();
supplTab.ExtractRows(rows);
return(supplTab);
}
public void ModeAtLeastOneGroup(IMatrixData mdata, Dictionary <string, List <string> > samples,
double minValid)
{
List <int> validRows = new List <int>();
for (int i = 0; i < mdata.RowCount; i++)
{
bool validNum = false;
for (int j = 0; j < mdata.ColumnCount; j++)
{
foreach (KeyValuePair <string, List <string> > entry in samples)
{
if (entry.Value.Contains(mdata.ColumnNames[j]))
{
if (mdata.Values.Get(i, j) >= minValid)
{
validNum = true;
}
}
}
}
if (validNum)
{
validRows.Add(i);
}
}
mdata.ExtractRows(validRows.ToArray());
}
public static void UniqueRows(this IMatrixData mdata, string[] ids, Func <double[], double> combineNumeric,
Func <string[], string> combineString, Func <string[][], string[]> combineCategory,
Func <double[][], double[]> combineMultiNumeric)
{
int[] order = ArrayUtils.Order(ids);
List <int> uniqueIdx = new List <int>();
string lastId = "";
List <int> idxsWithSameId = new List <int>();
foreach (int j in order)
{
string id = ids[j];
if (id == lastId)
{
idxsWithSameId.Add(j);
}
else
{
CombineRows(mdata, idxsWithSameId, combineNumeric, combineString, combineCategory, combineMultiNumeric);
uniqueIdx.Add(j);
idxsWithSameId.Clear();
idxsWithSameId.Add(j);
}
lastId = id;
}
CombineRows(mdata, idxsWithSameId, combineNumeric, combineString, combineCategory, combineMultiNumeric);
mdata.ExtractRows(uniqueIdx.ToArray());
}
public void ModeAllGroup(IMatrixData mdata, Dictionary <string, List <string> > samples,
double minValid)
{
List <int> validRows = new List <int>();
for (int i = 0; i < mdata.RowCount; i++)
{
Dictionary <string, int> detectNum = new Dictionary <string, int>();
for (int j = 0; j < mdata.ColumnCount; j++)
{
string preKey = "";
bool validNum = false;
foreach (KeyValuePair <string, List <string> > entry in samples)
{
if (!detectNum.ContainsKey(entry.Key))
{
detectNum.Add(entry.Key, 0);
}
if (entry.Value.Contains(mdata.ColumnNames[j]))
{
if (preKey == "")
{
preKey = entry.Key;
}
else if (preKey != "" && preKey != entry.Key)
{
if (validNum)
{
detectNum[entry.Key]++;
validNum = false;
}
}
if (mdata.Values.Get(i, j) >= minValid)
{
validNum = true;
}
preKey = entry.Key;
}
}
if (validNum)
{
detectNum[preKey]++;
}
}
bool fail = false;
foreach (KeyValuePair <string, int> entry in detectNum)
{
if (entry.Value == 0)
{
fail = true;
}
}
if (!fail)
{
validRows.Add(i);
}
}
mdata.ExtractRows(validRows.ToArray());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
var numberOfRows = param.GetParam <int>("Number of rows").Value;
var indices = Enumerable.Range(0, numberOfRows).ToArray();
mdata.ExtractRows(indices);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int lines = param.GetParam <int>("Number of rows").Value;
int[] remains = Enumerable.Range(0, lines).ToArray();
mdata.ExtractRows(remains);
}
private static void SortByValues <T>(IMatrixData mdata, T[] v, bool descending) where T : IComparable <T>
{
int[] o = ArrayUtils.Order(v);
if (descending)
{
ArrayUtils.Revert(o);
}
mdata.ExtractRows(o);
}
private void RemoveSelectedRowsButton_OnClick(object sender, EventArgs e)
{
int[] sel = tableView1.GetSelectedRows();
if (sel.Length == 0)
{
MessageBox.Show(Loc.PleaseSelectSomeRows);
}
IMatrixData mx = (IMatrixData)mdata.Clone();
mx.ExtractRows(ArrayUtils.Complement(sel, tableView1.RowCount));
createNewMatrix(mx);
}
private void KeepSelectedRowsButton_OnClick(object sender, RoutedEventArgs e)
{
int[] sel = TableView.GetSelectedRows();
if (sel.Length == 0)
{
MessageBox.Show("Please select some rows.");
}
IMatrixData mx = (IMatrixData)mdata.Clone();
mx.ExtractRows(sel);
createNewMatrix(mx);
}
private void KeepSelectedRowsButton_OnClick(object sender, EventArgs e)
{
int[] sel = tableView1.GetSelectedRows();
if (sel.Length == 0)
{
MessageBox.Show(Loc.PleaseSelectSomeRows);
}
IMatrixData mx = (IMatrixData)mdata.Clone();
mx.ExtractRows(sel);
createNewMatrix(mx);
}
private void RemoveSelectedRowsButton_OnClick(object sender, RoutedEventArgs e)
{
int[] sel = TableView.GetSelectedRows();
if (sel.Length == 0)
{
MessageBox.Show("Please select some rows.");
}
IMatrixData mx = (IMatrixData)mdata.Clone();
mx.ExtractRows(ArrayUtils.Complement(sel, TableView.RowCount));
createNewMatrix(mx);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int nrows = param.GetParam <int>("Number of rows").Value;
if (nrows >= mdata.RowCount)
{
return;
}
Random2 rand = new Random2(7);
int[] inds = rand.NextPermutation(mdata.RowCount).SubArray(nrows);
mdata.ExtractRows(inds);
}
public static void FilterRowsNew(IMatrixData mdata, Parameters parameters, int[] rows)
{
bool reduceMatrix = UnpackFilterModeParam(parameters) == FilterMode.Reduce;
if (parameters.GetParam <int>("Filter mode").Value == 0)
{
mdata.ExtractRows(rows);
}
else if (parameters.GetParam <int>("Filter mode").Value == 1)
{
Array.Sort(rows);
string[][] col = new string[mdata.RowCount][];
for (int i = 0; i < col.Length; i++)
{
bool contains = Array.BinarySearch(rows, i) >= 0;
col[i] = contains ? new[] { "Keep" } : new[] { "Discard" };
}
mdata.AddCategoryColumn("Filter", "", col);
}
else if (parameters.GetParam <int>("Filter mode").Value == 2)
{
mdata.ExtractRows(rows);
}
}
public static void FilterRows(IMatrixData mdata, Parameters parameters, int[] rows)
{
bool reduceMatrix = GetReduceMatrix(parameters);
if (reduceMatrix)
{
mdata.ExtractRows(rows);
}
else
{
Array.Sort(rows);
string[][] col = new string[mdata.RowCount][];
for (int i = 0; i < col.Length; i++)
{
bool contains = Array.BinarySearch(rows, i) >= 0;
col[i] = contains ? new[] { "Keep" } : new[] { "Discard" };
}
mdata.AddCategoryColumn("Filter", "", col);
}
}
public void RemoveLowValuesInRow(IMatrixData mdata, double[] minValids)
{
List <int> validRows = new List <int>();
for (int i = 0; i < mdata.RowCount; i++)
{
bool validNum = false;
for (int j = 0; j < mdata.ColumnCount; j++)
{
if (mdata.Values.Get(i, j) >= minValids[j])
{
validNum = true;
}
}
if (validNum)
{
validRows.Add(i);
}
}
mdata.ExtractRows(validRows.ToArray());
}
public static void FilterRows(IMatrixData mdata, Parameters parameters, int[] rows)
{
bool reduceMatrix = GetReduceMatrix(parameters);
if (reduceMatrix){
mdata.ExtractRows(rows);
} else{
Array.Sort(rows);
string[][] col = new string[mdata.RowCount][];
for (int i = 0; i < col.Length; i++){
bool contains = Array.BinarySearch(rows, i) >= 0;
col[i] = contains ? new[]{"Keep"} : new[]{"Discard"};
}
mdata.AddCategoryColumn("Filter", "", col);
}
}
public void ExtractValues(IMatrixData mdata, Dictionary <string, List <string> > samples,
ParameterWithSubParams <int> minValidType, Dictionary <string, int> minValidAmount, int type)
{
int minValid = -1;
List <int> totalCount = new List <int>();
if (minValidType.Value == 0)
{
minValid = minValidType.GetSubParameters().GetParam <int>("Min. raw counts").Value;
}
else
{
minValid = minValidType.GetSubParameters().GetParam <int>("Min. cpm").Value;
for (int j = 0; j < mdata.ColumnCount; j++)
{
int Count = 0;
foreach (int c in mdata.Values.GetColumn(j))
{
Count = Count + c;
}
totalCount.Add(Count);
}
}
List <int> validRows = new List <int>();
Dictionary <string, int> validNum = new Dictionary <string, int>();
foreach (KeyValuePair <string, List <string> > entry in samples)
{
validNum.Add(entry.Key, 0);
}
for (int i = 0; i < mdata.RowCount; i++)
{
int totalValids = 0;
foreach (KeyValuePair <string, List <string> > entry in samples)
{
validNum[entry.Key] = 0;
}
for (int j = 0; j < mdata.ColumnCount; j++)
{
foreach (KeyValuePair <string, List <string> > entry in samples)
{
if (entry.Value.Contains(mdata.ColumnNames[j]))
{
if (minValidType.Value == 0)
{
if (mdata.Values.Get(i, j) >= minValid)
{
validNum[entry.Key]++;
}
}
else
{
double cpm = (mdata.Values.Get(i, j) / totalCount[j]) * 1000000;
if (cpm >= minValid)
{
validNum[entry.Key]++;
}
}
}
}
}
foreach (KeyValuePair <string, int> entry in validNum)
{
if (validNum[entry.Key] >= minValidAmount[entry.Key])
{
totalValids++;
}
}
if ((type == 0) && (totalValids > 0))
{
validRows.Add(i);
}
else if ((type == 1) && (totalValids == samples.Count))
{
validRows.Add(i);
}
else if (type == 2)
{
int detectSample = 0;
foreach (KeyValuePair <string, int> entry in validNum)
{
detectSample = detectSample + validNum[entry.Key];
}
if (detectSample >= minValidAmount["total"])
{
validRows.Add(i);
}
}
}
mdata.ExtractRows(validRows.ToArray());
}