public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] cols = param.GetMultiChoiceParam("Columns").Value;
HashSet<int> w = ArrayUtils.ToHashSet(param.GetMultiChoiceParam("Calculate").Value);
bool[] include = new bool[SummaryStatisticsRows.procs.Length];
double[][] rowws = new double[SummaryStatisticsRows.procs.Length][];
for (int i = 0; i < include.Length; i++){
include[i] = w.Contains(i);
if (include[i]){
rowws[i] = new double[cols.Length];
}
}
for (int i = 0; i < cols.Length; i++){
double[] vals = GetColumn(cols[i], mdata);
for (int j = 0; j < include.Length; j++){
if (include[j]){
rowws[j][i] = SummaryStatisticsRows.procs[j].Item2(vals);
}
}
}
List<double[]> ex = new List<double[]>();
List<string> names = new List<string>();
for (int i = 0; i < include.Length; i++){
if (include[i]){
ex.Add(rowws[i]);
names.Add(SummaryStatisticsRows.procs[i].Item1);
}
}
float[,] exVals = GetExVals(ex);
IEnumerable<string> colNames = GetColNames(mdata, cols);
mdata.SetData("Summary", new List<string>(colNames), exVals, new List<string>(new[]{"Type"}),
new List<string[]>(new[]{names.ToArray()}), new List<string>(), new List<string[][]>(), new List<string>(),
new List<double[]>(), new List<string>(), new List<double[][]>());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
SingleChoiceWithSubParams access = param.GetSingleChoiceWithSubParams("Matrix access");
bool rows = access.Value == 0;
int groupInd;
if (rows) {
groupInd = access.GetSubParameters().GetSingleChoiceParam("Grouping").Value - 1;
} else {
groupInd = -1;
}
int what = param.GetSingleChoiceParam("Subtract what").Value;
if (groupInd < 0){
SubtractValues(rows, GetFunc(what), mdata);
} else{
string[][] catRow = mdata.CategoryRows[groupInd];
foreach (string[] t in catRow) {
if (t.Length > 1) {
processInfo.ErrString = "The groups are overlapping.";
return;
}
}
SubtractGroups(mdata, catRow, GetFunc(what));
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
bool rows = param.GetSingleChoiceParam("Matrix access").Value == 0;
double min = param.GetDoubleParam("Minimum").Value;
double max = param.GetDoubleParam("Maximum").Value;
MapToInterval1(rows, mdata, min, max);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int numQuantiles = param.GetIntParam("Number of quantiles").Value;
int[] colInds = param.GetMultiChoiceParam("Columns").Value;
foreach (int colInd in colInds){
float[] vals = mdata.GetExpressionColumn(colInd);
List<int> v = new List<int>();
for (int i = 0; i < vals.Length; i++){
if (!float.IsNaN(vals[i])){
v.Add(i);
}
}
int[] o = v.ToArray();
vals = ArrayUtils.SubArray(vals, o);
int[] q = ArrayUtils.Order(vals);
o = ArrayUtils.SubArray(o, q);
string[][] catCol = new string[mdata.RowCount][];
for (int i = 0; i < catCol.Length; i++){
catCol[i] = new[]{"missing"};
}
for (int i = 0; i < o.Length; i++){
int catVal = (i*numQuantiles)/o.Length + 1;
catCol[o[i]] = new[]{"Q" + catVal};
}
string name = mdata.ExpressionColumnNames[colInd] + "_q";
string desc = "The column " + mdata.ExpressionColumnNames[colInd] + " has been divided into " + numQuantiles +
" quantiles.";
mdata.AddCategoryColumn(name, desc, catCol);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
double width = param.GetDoubleParam("Width").Value;
double shift = param.GetDoubleParam("Down shift").Value;
bool separateColumns = param.GetSingleChoiceParam("Mode").Value == 0;
if (separateColumns){
ReplaceMissingsByGaussianByColumn(width, shift, mdata);
} else{
ReplaceMissingsByGaussianWholeMatrix(width, shift, mdata);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int colInd = param.GetSingleChoiceParam("Column").Value;
double value = param.GetDoubleParam("Value").Value;
int ruleInd = param.GetSingleChoiceParam("Remove if").Value;
bool keepNan = param.GetBoolParam("Keep NaN").Value;
double[] vals = colInd < mdata.NumericColumnCount
? mdata.NumericColumns[colInd] : ArrayUtils.ToDoubles(mdata.GetExpressionColumn(colInd - mdata.NumericColumnCount));
List<int> valids = new List<int>();
for (int i = 0; i < vals.Length; i++){
bool valid;
double val = vals[i];
if (double.IsNaN(val)){
valid = keepNan;
} else{
switch (ruleInd){
case 0:
valid = val > value;
break;
case 1:
valid = val >= value;
break;
case 2:
valid = val != value;
break;
case 3:
valid = val == value;
break;
case 4:
valid = val <= value;
break;
case 5:
valid = val < value;
break;
default:
throw new Exception("Never get here.");
}
}
if (valid){
valids.Add(i);
}
}
PerseusPluginUtils.FilterRows(mdata, param, valids.ToArray());
}
public IMatrixAnalysisResult AnalyzeData(IMatrixData mdata, Parameters param, ProcessInfo processInfo)
{
return new SelectRowsManuallyResult(mdata);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
SingleChoiceWithSubParams sp = param.GetSingleChoiceWithSubParams("Source type");
Parameters subParams = sp.GetSubParameters();
int[] colInds = subParams.GetMultiChoiceParam("Columns").Value;
int which = subParams.GetSingleChoiceParam("Target type").Value;
switch (sp.Value){
case 0:
ExpressionToNumeric(colInds, mdata);
break;
case 1:
if (which == 0){
NumericToCategorical(colInds, mdata);
} else{
NumericToExpression(colInds, mdata);
}
break;
case 2:
if (which == 0){
CategoricalToNumeric(colInds, mdata);
} else{
CategoricalToString(colInds, mdata);
}
break;
case 3:
StringToCategorical(colInds, mdata);
break;
default:
throw new Exception("Never get here");
}
}
public void LoadData(IMatrixData matrixData, Parameters parameters, ProcessInfo processInfo)
{
PerseusLoadFileParam par = (PerseusLoadFileParam) parameters.GetParam("File");
string filename = par.Filename;
if (string.IsNullOrEmpty(filename)){
processInfo.ErrString = "Please specify a filename";
return;
}
if (!File.Exists(filename)){
processInfo.ErrString = "File '" + filename + "' does not exist.";
return;
}
string[] colNames;
Dictionary<string, string[]> annotationRows = new Dictionary<string, string[]>();
try{
colNames = TabSep.GetColumnNames(filename, commentPrefix, commentPrefixExceptions, annotationRows);
} catch (Exception){
processInfo.ErrString = "Could not open the file '" + filename + "'. It is probably opened by another program.";
return;
}
string[] colDescriptions = null;
string[] colTypes = null;
bool[] colVisible = null;
if (annotationRows.ContainsKey("Description")){
colDescriptions = annotationRows["Description"];
annotationRows.Remove("Description");
}
if (annotationRows.ContainsKey("Type")){
colTypes = annotationRows["Type"];
annotationRows.Remove("Type");
}
if (annotationRows.ContainsKey("Visible")){
string[] colVis = annotationRows["Visible"];
colVisible = new bool[colVis.Length];
for (int i = 0; i < colVisible.Length; i++){
colVisible[i] = bool.Parse(colVis[i]);
}
annotationRows.Remove("Visible");
}
int[] eInds = par.ExpressionColumnIndices;
int[] cInds = par.CategoryColumnIndices;
int[] nInds = par.NumericalColumnIndices;
int[] tInds = par.TextColumnIndices;
int[] mInds = par.MultiNumericalColumnIndices;
int[] allInds = ArrayUtils.Concat(new[]{eInds, cInds, nInds, tInds, mInds});
Array.Sort(allInds);
for (int i = 0; i < allInds.Length - 1; i++){
if (allInds[i + 1] == allInds[i]){
processInfo.ErrString = "Column '" + colNames[allInds[i]] + "' has been selected multiple times";
return;
}
}
string[] allColNames = ArrayUtils.SubArray(colNames, allInds);
Array.Sort(allColNames);
for (int i = 0; i < allColNames.Length - 1; i++){
if (allColNames[i + 1].Equals(allColNames[i])){
processInfo.ErrString = "Column name '" + allColNames[i] + "' occurs multiple times.";
return;
}
}
LoadData(colNames, colDescriptions, eInds, cInds, nInds, tInds, mInds, filename, matrixData, annotationRows,
processInfo.Progress, processInfo.Status);
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] multiNumCols = param1.GetMultiChoiceParam("Multi-numeric columns").Value;
Array.Sort(multiNumCols);
int[] stringCols = param1.GetMultiChoiceParam("String columns").Value;
Array.Sort(stringCols);
HashSet<int> multinumCols2 = new HashSet<int>(multiNumCols);
HashSet<int> stringCols2 = new HashSet<int>(stringCols);
if (multiNumCols.Length + stringCols.Length == 0){
processInfo.ErrString = "Please select some columns.";
return;
}
int rowCount = GetNewRowCount(mdata, multiNumCols, stringCols);
float[,] expVals = new float[rowCount,mdata.ExpressionColumnCount];
List<string[]> stringC = new List<string[]>();
for (int i = 0; i < mdata.StringColumnCount; i++){
stringC.Add(new string[rowCount]);
}
List<double[]> numC = new List<double[]>();
for (int i = 0; i < mdata.NumericColumnCount; i++){
numC.Add(new double[rowCount]);
}
List<string[][]> catC = new List<string[][]>();
for (int i = 0; i < mdata.CategoryColumnCount; i++){
catC.Add(new string[rowCount][]);
}
List<double[][]> multiNumC = new List<double[][]>();
for (int i = 0; i < mdata.MultiNumericColumnCount; i++){
multiNumC.Add(new double[rowCount][]);
}
int count = 0;
for (int i = 0; i < mdata.RowCount; i++){
string err;
int entryCount = GetEntryCount(i, mdata, multiNumCols, stringCols, out err);
if (err != null){
processInfo.ErrString = err;
return;
}
bool empty = entryCount == 0;
entryCount = Math.Max(entryCount, 1);
for (int j = 0; j < entryCount; j++){
for (int k = 0; k < mdata.ExpressionColumnCount; k++){
expVals[count + j, k] = mdata[i, k];
}
for (int k = 0; k < mdata.NumericColumnCount; k++){
numC[k][count + j] = mdata.NumericColumns[k][i];
}
for (int k = 0; k < mdata.CategoryColumnCount; k++){
catC[k][count + j] = mdata.CategoryColumns[k][i];
}
}
for (int k = 0; k < mdata.MultiNumericColumnCount; k++){
if (multinumCols2.Contains(k)){
if (empty){
multiNumC[k][count] = new double[0];
} else{
double[] vals = mdata.MultiNumericColumns[k][i];
for (int j = 0; j < entryCount; j++){
multiNumC[k][count + j] = new[]{vals[j]};
}
}
} else{
for (int j = 0; j < entryCount; j++){
multiNumC[k][count + j] = mdata.MultiNumericColumns[k][i];
}
}
}
for (int k = 0; k < mdata.StringColumnCount; k++){
if (stringCols2.Contains(k)){
if (empty){
stringC[k][count] = "";
} else{
string[] vals = mdata.StringColumns[k][i].Split(';');
for (int j = 0; j < entryCount; j++){
stringC[k][count + j] = vals[j];
}
}
} else{
for (int j = 0; j < entryCount; j++){
stringC[k][count + j] = mdata.StringColumns[k][i];
}
}
}
count += entryCount;
}
int[] multiNumComplement = ArrayUtils.Complement(multiNumCols, mdata.MultiNumericColumnCount);
List<double[][]> toBeTransformed = ArrayUtils.SubList(multiNumC, multiNumCols);
multiNumC = ArrayUtils.SubList(multiNumC, multiNumComplement);
foreach (double[][] d in toBeTransformed){
numC.Add(Transform(d));
}
mdata.SetData(mdata.Name, mdata.ExpressionColumnNames, expVals, mdata.StringColumnNames, stringC,
mdata.CategoryColumnNames, catC,
new List<string>(ArrayUtils.Concat(mdata.NumericColumnNames,
ArrayUtils.SubList(mdata.MultiNumericColumnNames, multiNumCols))), numC,
new List<string>(ArrayUtils.SubArray(mdata.MultiNumericColumnNames, multiNumComplement)), multiNumC);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
SingleChoiceParam access = param.GetSingleChoiceParam("Matrix access");
bool rows = access.Value == 0;
int what = param.GetSingleChoiceParam("Divide by what").Value;
DivideImpl(rows, ArrayUtils.Mean, mdata);
switch (what){
case 0:
DivideImpl(rows, ArrayUtils.Mean, mdata);
break;
case 1:
DivideImpl(rows, ArrayUtils.Median, mdata);
break;
case 2:
DivideImpl(rows, ArrayUtils.MostFrequentValue, mdata);
break;
default:
throw new Exception("Never get here.");
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int ind = param.GetSingleChoiceParam("Column").Value;
bool descending = param.GetBoolParam("Descending").Value;
if (ind < mdata.ExpressionColumnCount){
float[] v = mdata.GetExpressionColumn(ind);
int[] o = ArrayUtils.Order(v);
if (descending){
ArrayUtils.Revert(o);
}
mdata.ExtractExpressionRows(o);
} else{
double[] v = mdata.NumericColumns[ind - mdata.ExpressionColumnCount];
int[] o = ArrayUtils.Order(v);
if (descending){
ArrayUtils.Revert(o);
}
mdata.ExtractExpressionRows(o);
}
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] cols = param1.GetMultiChoiceParam("Columns").Value;
int[] ops = param1.GetMultiChoiceParam("Operation").Value;
foreach (int t in ops){
double[][] vals = new double[cols.Length][];
for (int i = 0; i < cols.Length; i++){
double[][] x = mdata.MultiNumericColumns[cols[i]];
vals[i] = new double[x.Length];
for (int j = 0; j < vals[i].Length; j++){
vals[i][j] = operations[t](x[j]);
}
}
for (int i = 0; i < cols.Length; i++){
mdata.AddNumericColumn(mdata.MultiNumericColumnNames[cols[i]] + "_" + names[t], "", vals[i]);
}
}
}
public void Export(Parameters parameters, IMatrixData data, ProcessInfo processInfo)
{
string filename = parameters.GetFileParam("File name").Value;
StreamWriter writer;
try{
writer = new StreamWriter(filename);
} catch (Exception e){
processInfo.ErrString = e.Message;
return;
}
List<string> words = new List<string>();
for (int i = 0; i < data.ExpressionColumnCount; i++){
words.Add(Trunc(data.ExpressionColumnNames[i]));
}
for (int i = 0; i < data.CategoryColumnCount; i++){
words.Add(Trunc(data.CategoryColumnNames[i]));
}
for (int i = 0; i < data.NumericColumnCount; i++){
words.Add(Trunc(data.NumericColumnNames[i]));
}
for (int i = 0; i < data.StringColumnCount; i++){
words.Add(Trunc(data.StringColumnNames[i]));
}
for (int i = 0; i < data.MultiNumericColumnCount; i++){
words.Add(Trunc(data.MultiNumericColumnNames[i]));
}
writer.WriteLine(StringUtils.Concat("\t", words));
if (HasAnyDescription(data)){
words = new List<string>();
for (int i = 0; i < data.ExpressionColumnCount; i++){
words.Add(Trunc(data.ExpressionColumnDescriptions[i] ?? ""));
}
for (int i = 0; i < data.CategoryColumnCount; i++){
words.Add(Trunc(data.CategoryColumnDescriptions[i] ?? ""));
}
for (int i = 0; i < data.NumericColumnCount; i++){
words.Add(Trunc(data.NumericColumnDescriptions[i] ?? ""));
}
for (int i = 0; i < data.StringColumnCount; i++){
words.Add(Trunc(data.StringColumnDescriptions[i] ?? ""));
}
for (int i = 0; i < data.MultiNumericColumnCount; i++){
words.Add(Trunc(data.MultiNumericColumnDescriptions[i] ?? ""));
}
writer.WriteLine("#!{Description}" + StringUtils.Concat("\t", words));
}
words = new List<string>();
for (int i = 0; i < data.ExpressionColumnCount; i++){
words.Add("E");
}
for (int i = 0; i < data.CategoryColumnCount; i++){
words.Add("C");
}
for (int i = 0; i < data.NumericColumnCount; i++){
words.Add("N");
}
for (int i = 0; i < data.StringColumnCount; i++){
words.Add("T");
}
for (int i = 0; i < data.MultiNumericColumnCount; i++){
words.Add("M");
}
writer.WriteLine("#!{Type}" + StringUtils.Concat("\t", words));
for (int i = 0; i < data.NumericRowCount; i++){
words = new List<string>();
for (int j = 0; j < data.ExpressionColumnCount; j++){
words.Add("" + data.NumericRows[i][j]);
}
for (int j = 0; j < data.CategoryColumnCount; j++){
words.Add("");
}
for (int j = 0; j < data.NumericColumnCount; j++){
words.Add("");
}
for (int j = 0; j < data.StringColumnCount; j++){
words.Add("");
}
for (int j = 0; j < data.MultiNumericColumnCount; j++){
words.Add("");
}
writer.WriteLine("#!{N:" + data.NumericRowNames[i] + "}" + StringUtils.Concat("\t", words));
}
for (int i = 0; i < data.CategoryRowCount; i++){
words = new List<string>();
for (int j = 0; j < data.ExpressionColumnCount; j++){
string[] s = data.CategoryRows[i][j];
words.Add(s.Length == 0 ? "" : StringUtils.Concat(";", s));
}
for (int j = 0; j < data.CategoryColumnCount; j++){
words.Add("");
}
for (int j = 0; j < data.NumericColumnCount; j++){
words.Add("");
}
for (int j = 0; j < data.StringColumnCount; j++){
words.Add("");
}
for (int j = 0; j < data.MultiNumericColumnCount; j++){
words.Add("");
}
writer.WriteLine("#!{C:" + data.CategoryRowNames[i] + "}" + StringUtils.Concat("\t", words));
}
for (int j = 0; j < data.RowCount; j++){
words = new List<string>();
for (int i = 0; i < data.ExpressionColumnCount; i++){
words.Add(Trunc("" + data[j, i]));
}
for (int i = 0; i < data.CategoryColumnCount; i++){
string[] q = data.CategoryColumns[i][j] ?? new string[0];
words.Add(Trunc((q.Length > 0 ? StringUtils.Concat(";", q) : "")));
}
for (int i = 0; i < data.NumericColumnCount; i++){
words.Add(Trunc("" + data.NumericColumns[i][j]));
}
for (int i = 0; i < data.StringColumnCount; i++){
words.Add(Trunc(data.StringColumns[i][j]));
}
for (int i = 0; i < data.MultiNumericColumnCount; i++){
double[] q = data.MultiNumericColumns[i][j];
words.Add(Trunc((q.Length > 0 ? StringUtils.Concat(";", q) : "")));
}
string s = StringUtils.Concat("\t", words);
s = s.Replace("\"", "");
writer.WriteLine(s);
}
writer.Close();
}
public void ProcessData(IMatrixData data, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] exColInds = GetValidExCols(data);
int[] numColInds = GetValidNumCols(data);
int[] multiNumColInds = GetValidMultiNumCols(data);
int[] catColInds = GetValidCatCols(data);
int[] textColInds = GetValidTextCols(data);
if (exColInds.Length < data.ExpressionColumnCount) {
data.ExtractExpressionColumns(exColInds);
}
if (numColInds.Length < data.NumericColumnCount) {
data.NumericColumns = ArrayUtils.SubList(data.NumericColumns, numColInds);
data.NumericColumnNames = ArrayUtils.SubList(data.NumericColumnNames, numColInds);
data.NumericColumnDescriptions = ArrayUtils.SubList(data.NumericColumnDescriptions, numColInds);
}
if (multiNumColInds.Length < data.MultiNumericColumnCount) {
data.MultiNumericColumns = ArrayUtils.SubList(data.MultiNumericColumns, multiNumColInds);
data.MultiNumericColumnNames = ArrayUtils.SubList(data.MultiNumericColumnNames, multiNumColInds);
data.MultiNumericColumnDescriptions = ArrayUtils.SubList(data.MultiNumericColumnDescriptions, multiNumColInds);
}
if (catColInds.Length < data.CategoryColumnCount) {
data.CategoryColumns = ArrayUtils.SubList(data.CategoryColumns, catColInds);
data.CategoryColumnNames = ArrayUtils.SubList(data.CategoryColumnNames, catColInds);
data.CategoryColumnDescriptions = ArrayUtils.SubList(data.CategoryColumnDescriptions, catColInds);
}
if (textColInds.Length < data.StringColumnCount) {
data.StringColumns = ArrayUtils.SubList(data.StringColumns, textColInds);
data.StringColumnNames = ArrayUtils.SubList(data.StringColumnNames, textColInds);
data.StringColumnDescriptions = ArrayUtils.SubList(data.StringColumnDescriptions, textColInds);
}
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] stringCols = param1.GetMultiChoiceParam("String columns").Value;
int[] catCols = param1.GetMultiChoiceParam("Categorical columns").Value;
if (catCols.Length + stringCols.Length == 0){
processInfo.ErrString = "Please select some columns.";
return;
}
foreach (string[][] col in catCols.Select(catCol => mdata.CategoryColumns[catCol])){
for (int i = 0; i < col.Length; i++){
col[i] = ArrayUtils.UniqueValues(col[i]);
}
}
foreach (string[] col in stringCols.Select(stringCol => mdata.StringColumns[stringCol])){
for (int i = 0; i < col.Length; i++){
string q = col[i];
if (q.Length == 0){
continue;
}
string[] w = q.Split(';');
w = ArrayUtils.UniqueValues(w);
col[i] = StringUtils.Concat(";", w);
}
}
}
public void ProcessData(IMatrixData data, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] exColInds = param.GetMultiChoiceParam("Expression columns").Value;
int[] numColInds = param.GetMultiChoiceParam("Numerical columns").Value;
int[] multiNumColInds = param.GetMultiChoiceParam("Multi-numerical columns").Value;
int[] catColInds = param.GetMultiChoiceParam("Categorical columns").Value;
int[] textColInds = param.GetMultiChoiceParam("Text columns").Value;
data.ExtractExpressionColumns(exColInds);
data.NumericColumns = ArrayUtils.SubList(data.NumericColumns, numColInds);
data.NumericColumnNames = ArrayUtils.SubList(data.NumericColumnNames, numColInds);
data.NumericColumnDescriptions = ArrayUtils.SubList(data.NumericColumnDescriptions, numColInds);
data.MultiNumericColumns = ArrayUtils.SubList(data.MultiNumericColumns, multiNumColInds);
data.MultiNumericColumnNames = ArrayUtils.SubList(data.MultiNumericColumnNames, multiNumColInds);
data.MultiNumericColumnDescriptions = ArrayUtils.SubList(data.MultiNumericColumnDescriptions, multiNumColInds);
data.CategoryColumns = ArrayUtils.SubList(data.CategoryColumns, catColInds);
data.CategoryColumnNames = ArrayUtils.SubList(data.CategoryColumnNames, catColInds);
data.CategoryColumnDescriptions = ArrayUtils.SubList(data.CategoryColumnDescriptions, catColInds);
data.StringColumns = ArrayUtils.SubList(data.StringColumns, textColInds);
data.StringColumnNames = ArrayUtils.SubList(data.StringColumnNames, textColInds);
data.StringColumnDescriptions = ArrayUtils.SubList(data.StringColumnDescriptions, textColInds);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int colInd = param.GetSingleChoiceParam("Column").Value;
string value = param.GetStringParam("Find").Value;
bool remove = param.GetSingleChoiceParam("Mode").Value == 0;
string[][] cats = mdata.CategoryColumns[colInd];
List<int> valids = new List<int>();
for (int i = 0; i < cats.Length; i++){
bool valid = true;
foreach (string w in cats[i]){
if (w.Equals(value)){
valid = false;
break;
}
}
if ((valid && remove) || (!valid && !remove)){
valids.Add(i);
}
}
mdata.ExtractExpressionRows(valids.ToArray());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
string[][] col = mdata.CategoryColumns[param.GetSingleChoiceParam("Indicator column").Value];
string term = param.GetStringParam("Value").Value;
List<int> inds = new List<int>();
for (int i = 0; i < col.Length; i++){
if (Contains(col[i], term)){
inds.Add(i);
}
}
float[][] profiles = new float[inds.Count][];
for (int i = 0; i < profiles.Length; i++){
profiles[i] = mdata.GetExpressionRow(inds[i]);
float mean = (float) ArrayUtils.Mean(profiles[i]);
for (int j = 0; j < profiles[i].Length; j++){
profiles[i][j] -= mean;
}
}
float[] totalProfile = new float[mdata.ExpressionColumnCount];
for (int i = 0; i < totalProfile.Length; i++){
List<float> vals = new List<float>();
foreach (float[] t in profiles){
float val = t[i];
if (float.IsNaN(val) || float.IsInfinity(val)){
continue;
}
vals.Add(val);
}
totalProfile[i] = vals.Count > 0 ? ArrayUtils.Median(vals) : float.NaN;
}
for (int i = 0; i < mdata.RowCount; i++){
for (int j = 0; j < mdata.ExpressionColumnCount; j++){
mdata[i, j] -= totalProfile[j];
}
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
bool rows = param.GetSingleChoiceParam("Matrix access").Value == 0;
bool atLeast = param.GetSingleChoiceParam("Side").Value == 0;
int numValids = param.GetIntParam("Number of valid values").Value;
SingleChoiceWithSubParams modeParam = param.GetSingleChoiceWithSubParams("Mode");
int modeInd = modeParam.Value;
if (modeInd != 0 && mdata.CategoryRowNames.Count == 0){
processInfo.ErrString = "No grouping is defined.";
return;
}
if (modeInd != 0 && !rows){
processInfo.ErrString = "Group-wise filtering can only be appled to rows.";
return;
}
if (modeInd != 0){
int gind = modeParam.GetSubParameters().GetSingleChoiceParam("Grouping").Value;
string[][] groupCol = mdata.CategoryRows[gind];
ValidValueFilterGroup(numValids, mdata, param, modeInd == 2, groupCol, atLeast);
} else{
ValidValueFilter1(rows, numValids, mdata, param, atLeast);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int nameCol = param.GetSingleChoiceParam("New column names").Value;
float[,] x = ArrayUtils.Transpose(mdata.ExpressionValues);
List<string> colNames = new List<string>(mdata.StringColumns[nameCol]);
List<string> rowNames = mdata.ExpressionColumnNames;
mdata.SetData(mdata.Name, colNames, x, new List<string>(new[]{"Name"}), new List<string[]>(new[]{rowNames.ToArray()}),
new List<string>(), new List<string[][]>(), new List<string>(), new List<double[]>(), new List<string>(),
new List<double[][]>());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
SingleChoiceParam access = param.GetSingleChoiceParam("Matrix access");
bool rows = access.Value == 0;
UnitVectors(rows, mdata);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
SingleChoiceWithSubParams xp = param.GetSingleChoiceWithSubParams("Expression column selection");
bool groups = xp.Value == 2;
string[] groupNames = null;
int[][] colIndsGroups = null;
if (groups){
int groupRowInd = xp.GetSubParameters().GetSingleChoiceParam("Group").Value;
string[][] groupCol = mdata.CategoryRows[groupRowInd];
groupNames = ArrayUtils.UniqueValuesPreserveOrder(groupCol);
colIndsGroups = PerseusPluginUtils.GetExpressionColIndices(groupCol, groupNames);
}
int[] useCols = xp.Value == 1
? xp.GetSubParameters().GetMultiChoiceParam("Columns").Value
: ArrayUtils.ConsecutiveInts(mdata.ExpressionColumnCount);
HashSet<int> w = ArrayUtils.ToHashSet(param.GetMultiChoiceParam("Calculate").Value);
bool[] include = new bool[procs.Length];
double[][] columns = new double[procs.Length][];
double[][][] columnsG = null;
if (groups){
columnsG = new double[procs.Length][][];
for (int i = 0; i < columnsG.Length; i++){
columnsG[i] = new double[groupNames.Length][];
}
}
for (int i = 0; i < include.Length; i++){
include[i] = w.Contains(i);
if (include[i]){
columns[i] = new double[mdata.RowCount];
if (groups){
for (int j = 0; j < groupNames.Length; j++){
columnsG[i][j] = new double[mdata.RowCount];
}
}
}
}
for (int i = 0; i < mdata.RowCount; i++){
List<double> v = new List<double>();
foreach (int j in useCols){
double x = mdata[i, j];
if (!double.IsNaN(x) && !double.IsInfinity(x)){
v.Add(x);
}
}
for (int j = 0; j < include.Length; j++){
if (include[j]){
columns[j][i] = procs[j].Item2(v);
}
}
if (groups){
List<double>[] vg = new List<double>[groupNames.Length];
for (int j = 0; j < colIndsGroups.Length; j++){
vg[j] = new List<double>();
for (int k = 0; k < colIndsGroups[j].Length; k++){
double x = mdata[i, colIndsGroups[j][k]];
if (!double.IsNaN(x) && !double.IsInfinity(x)){
vg[j].Add(x);
}
}
}
for (int j = 0; j < include.Length; j++){
if (include[j]){
for (int k = 0; k < groupNames.Length; k++){
columnsG[j][k][i] = procs[j].Item2(vg[k]);
}
}
}
}
}
for (int i = 0; i < include.Length; i++){
if (include[i]){
mdata.AddNumericColumn(procs[i].Item1, procs[i].Item3, columns[i]);
if (groups){
for (int k = 0; k < groupNames.Length; k++){
mdata.AddNumericColumn(procs[i].Item1 + " " + groupNames[k], procs[i].Item3, columnsG[i][k]);
}
}
}
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int colIndex = param.GetSingleChoiceParam("Control column").Value;
if (colIndex < mdata.ExpressionColumnCount){
DivideByColumn(mdata, colIndex);
} else{
DivideByColumnNum(mdata, colIndex - mdata.ExpressionColumnCount);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] colIndx = param.GetMultiChoiceParam("Column 1").Value;
int[] colIndy = param.GetMultiChoiceParam("Column 2").Value;
if (colIndx.Length == 0){
processInfo.ErrString = "Please select some columns";
return;
}
if (colIndx.Length != colIndy.Length){
processInfo.ErrString = "Please select the same number of columns in the boxes for the first and second columns.";
return;
}
int points = param.GetIntParam("Number of points").Value;
for (int k = 0; k < colIndx.Length; k++){
float[] xvals = GetColumn(mdata, colIndx[k]);
float[] yvals = GetColumn(mdata, colIndy[k]);
float[] xvals1;
float[] yvals1;
NumUtils.GetValidPairs(xvals, yvals, out xvals1, out yvals1);
double xmin;
double xmax;
double ymin;
double ymax;
DensityEstimation.CalcRanges(xvals1, yvals1, out xmin, out xmax, out ymin, out ymax);
float[,] values = DensityEstimation.GetValuesOnGrid(xvals1, xmin, (xmax - xmin) / points, points, yvals1, ymin,
(ymax - ymin)/points, points);
DensityEstimation.DivideByMaximum(values);
//if (modeInd == 1){
// values = InvertRows(values);
// List<string> colNames = new List<string>();
// for (int i = 0; i < values.GetLength(1); i++){
// colNames.Add("" + i);
// }
// mdata.SetData(mdata.Name, colNames, values, new List<string>(), new List<string[]>(), new List<string>(),
// new List<string[][]>(), new List<string>(), new List<double[]>(), new List<string>(), new List<double[][]>());
// return;
//}
double[] xmat = new double[points];
for (int i = 0; i < points; i++){
xmat[i] = xmin + i*(xmax - xmin)/points;
}
double[] ymat = new double[points];
for (int i = 0; i < points; i++){
ymat[i] = ymin + i*(ymax - ymin)/points;
}
float[,] percvalues = CalcExcludedPercentage(values);
double[] dvals = new double[xvals.Length];
double[] pvals = new double[xvals.Length];
for (int i = 0; i < dvals.Length; i++){
double xx = xvals[i];
double yy = yvals[i];
if (!double.IsNaN(xx) && !double.IsNaN(yy)){
int xind = ArrayUtils.ClosestIndex(xmat, xx);
int yind = ArrayUtils.ClosestIndex(ymat, yy);
dvals[i] = values[xind, yind];
pvals[i] = percvalues[xind, yind];
} else{
dvals[i] = double.NaN;
pvals[i] = double.NaN;
}
}
string xname = GetColumnName(mdata, colIndx[k]);
string yname = GetColumnName(mdata, colIndy[k]);
mdata.AddNumericColumn("Density_" + xname + "_" + yname,
"Density of data points in the plane spanned by the columns " + xname + " and " + yname + ".", dvals);
mdata.AddNumericColumn("Excluded fraction_" + xname + "_" + yname,
"Percentage of points with a point density smaller than at this point in the plane spanned by the columns " + xname +
" and " + yname + ".", pvals);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
float value = (float) param.GetDoubleParam("Value").Value;
ReplaceMissingsByVal(value, mdata);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
float[,] vals = mdata.ExpressionValues;
double[] dm = new double[mdata.ExpressionColumnCount];
double[] dp = new double[mdata.ExpressionColumnCount];
for (int i = 0; i < mdata.ExpressionColumnCount; i++){
List<float> v = new List<float>();
foreach (float f in mdata.GetExpressionColumn(i)){
if (!float.IsNaN(f) && !float.IsInfinity(f)){
v.Add(f);
}
}
float[] d = v.ToArray();
float[] q = ArrayUtils.Quantiles(d, new[]{0.25, 0.5, 0.75});
for (int j = 0; j < mdata.RowCount; j++){
vals[j, i] -= q[1];
}
dm[i] = q[1] - q[0];
dp[i] = q[2] - q[1];
}
double adm = ArrayUtils.Median(dm);
double adp = ArrayUtils.Median(dp);
for (int i = 0; i < mdata.ExpressionColumnCount; i++){
for (int j = 0; j < mdata.RowCount; j++){
if (vals[j, i] < 0){
vals[j, i] = (float) (vals[j, i]*adm/dm[i]);
} else{
vals[j, i] = (float) (vals[j, i]*adp/dp[i]);
}
}
}
}
public IMatrixData CombineData(IMatrixData matrixData1, IMatrixData matrixData2, Parameters parameters,
ProcessInfo processInfo)
{
bool indicator = parameters.GetBoolParam("Indicator").Value;
int otherCol = parameters.GetSingleChoiceParam("Matching column 2").Value;
Average avExpression = GetAveraging(parameters.GetSingleChoiceParam("Combine expression values").Value);
Average avNumerical = GetAveraging(parameters.GetSingleChoiceParam("Combine numerical values").Value);
string[] q = matrixData2.StringColumns[otherCol];
string[][] w = new string[q.Length][];
for (int i = 0; i < q.Length; i++){
string r = q[i].Trim();
w[i] = r.Length == 0 ? new string[0] : r.Split(';');
w[i] = ArrayUtils.UniqueValues(w[i]);
}
Dictionary<string, List<int>> id2Cols = new Dictionary<string, List<int>>();
for (int i = 0; i < w.Length; i++){
foreach (string s in w[i]){
if (!id2Cols.ContainsKey(s)){
id2Cols.Add(s, new List<int>());
}
id2Cols[s].Add(i);
}
}
int pgCol = parameters.GetSingleChoiceParam("Matching column 1").Value;
string[] d = matrixData1.StringColumns[pgCol];
string[][] x = new string[d.Length][];
for (int i = 0; i < d.Length; i++){
string r = d[i].Trim();
x[i] = r.Length == 0 ? new string[0] : r.Split(';');
x[i] = ArrayUtils.UniqueValues(x[i]);
}
int[][] indexMap = new int[x.Length][];
string[][] indicatorCol = new string[x.Length][];
for (int i = 0; i < indexMap.Length; i++){
List<int> qwer = new List<int>();
foreach (string s in x[i]){
if (id2Cols.ContainsKey(s)){
List<int> en = id2Cols[s];
qwer.AddRange(en);
}
}
indexMap[i] = qwer.ToArray();
indexMap[i] = ArrayUtils.UniqueValues(indexMap[i]);
indicatorCol[i] = indexMap[i].Length > 0 ? new[]{"+"} : new string[0];
}
IMatrixData result = matrixData1.Copy();
SetAnnotationRows(result, matrixData1, matrixData2);
if (indicator){
result.AddCategoryColumn(matrixData2.Name, "", indicatorCol);
}
{
int[] exCols = parameters.GetMultiChoiceParam("Expression columns").Value;
float[,] newExColumns = new float[matrixData1.RowCount, exCols.Length];
float[,] newQuality = new float[matrixData1.RowCount, exCols.Length];
bool[,] newIsImputed = new bool[matrixData1.RowCount, exCols.Length];
string[] newExColNames = new string[exCols.Length];
float[,] oldEx = matrixData2.ExpressionValues;
float[,] oldQual = matrixData2.QualityValues;
bool[,] oldImp = matrixData2.IsImputed;
for (int i = 0; i < exCols.Length; i++) {
newExColNames[i] = matrixData2.ExpressionColumnNames[exCols[i]];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<double> values = new List<double>();
List<double> qual = new List<double>();
List<bool> imp = new List<bool>();
foreach (int ind in inds) {
double v = oldEx[ind, exCols[i]];
if (!double.IsNaN(v) && !double.IsInfinity(v)){
values.Add(v);
double qx = oldQual[ind, exCols[i]];
if (!double.IsNaN(qx) && !double.IsInfinity(qx)){
qual.Add(qx);
}
bool isi = oldImp[ind, exCols[i]];
imp.Add(isi);
}
}
newExColumns[j, i] = values.Count == 0 ? float.NaN : (float)avExpression(values.ToArray());
newQuality[j, i] = qual.Count == 0 ? float.NaN : (float)avExpression(qual.ToArray());
newIsImputed[j, i] = imp.Count != 0 && AvImp(imp.ToArray());
}
}
MakeNewNames(newExColNames, result.ExpressionColumnNames);
AddExpressionColumns(result, newExColNames, newExColumns, newQuality, newIsImputed);
}
{
int[] numCols = parameters.GetMultiChoiceParam("Numerical columns").Value;
double[][] newNumericalColumns = new double[numCols.Length][];
string[] newNumColNames = new string[numCols.Length];
for (int i = 0; i < numCols.Length; i++){
double[] oldCol = matrixData2.NumericColumns[numCols[i]];
newNumColNames[i] = matrixData2.NumericColumnNames[numCols[i]];
newNumericalColumns[i] = new double[matrixData1.RowCount];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<double> values = new List<double>();
foreach (int ind in inds){
double v = oldCol[ind];
if (!double.IsNaN(v)){
values.Add(v);
}
}
newNumericalColumns[i][j] = values.Count == 0 ? double.NaN : avNumerical(values.ToArray());
}
}
for (int i = 0; i < numCols.Length; i++){
result.AddNumericColumn(newNumColNames[i], "", newNumericalColumns[i]);
}
}
{
int[] catCols = parameters.GetMultiChoiceParam("Categorical columns").Value;
string[][][] newCatColumns = new string[catCols.Length][][];
string[] newCatColNames = new string[catCols.Length];
for (int i = 0; i < catCols.Length; i++){
string[][] oldCol = matrixData2.CategoryColumns[catCols[i]];
newCatColNames[i] = matrixData2.CategoryColumnNames[catCols[i]];
newCatColumns[i] = new string[matrixData1.RowCount][];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<string[]> values = new List<string[]>();
foreach (int ind in inds){
string[] v = oldCol[ind];
if (v.Length > 0){
values.Add(v);
}
}
newCatColumns[i][j] = values.Count == 0
? new string[0] : ArrayUtils.UniqueValues(ArrayUtils.Concat(values.ToArray()));
}
}
for (int i = 0; i < catCols.Length; i++){
result.AddCategoryColumn(newCatColNames[i], "", newCatColumns[i]);
}
}
{
int[] stringCols = parameters.GetMultiChoiceParam("String columns").Value;
string[][] newStringColumns = new string[stringCols.Length][];
string[] newStringColNames = new string[stringCols.Length];
for (int i = 0; i < stringCols.Length; i++){
string[] oldCol = matrixData2.StringColumns[stringCols[i]];
newStringColNames[i] = matrixData2.StringColumnNames[stringCols[i]];
newStringColumns[i] = new string[matrixData1.RowCount];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<string> values = new List<string>();
foreach (int ind in inds){
string v = oldCol[ind];
if (v.Length > 0){
values.Add(v);
}
}
newStringColumns[i][j] = values.Count == 0 ? "" : StringUtils.Concat(";", values.ToArray());
}
}
for (int i = 0; i < stringCols.Length; i++){
result.AddStringColumn(newStringColNames[i], "", newStringColumns[i]);
}
}
result.Origin = "Combination";
return result;
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
SingleChoiceWithSubParams scwsp = param.GetSingleChoiceWithSubParams("Action");
Parameters spar = scwsp.GetSubParameters();
switch (scwsp.Value){
case 0:
ProcessDataCreate(mdata, spar);
break;
case 1:
ProcessDataEdit(mdata, spar);
break;
case 2:
ProcessDataRename(mdata, spar);
break;
case 3:
ProcessDataDelete(mdata, spar);
break;
}
}
