public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] cols = param.GetParam <int[]>("Columns").Value;
int truncIndex = param.GetParam <int>("Use for truncation").Value;
TestTruncation truncation = truncIndex == 0
? TestTruncation.Pvalue
: (truncIndex == 1 ? TestTruncation.BenjaminiHochberg : TestTruncation.PermutationBased);
double threshold = param.GetParam <double>("Threshold value").Value;
int sideInd = param.GetParam <int>("Side").Value;
TestSide side;
switch (sideInd)
{
case 0:
side = TestSide.Both;
break;
case 1:
side = TestSide.Left;
break;
case 2:
side = TestSide.Right;
break;
default:
throw new Exception("Never get here.");
}
foreach (int col in cols)
{
BaseVector r = mdata.Values.GetColumn(col);
double[] pvals = CalcSignificanceA(r, side);
string[][] fdr;
switch (truncation)
{
case TestTruncation.Pvalue:
fdr = PerseusPluginUtils.CalcPvalueSignificance(pvals, threshold);
break;
case TestTruncation.BenjaminiHochberg:
double[] fdrs;
fdr = PerseusPluginUtils.CalcBenjaminiHochbergFdr(pvals, threshold, pvals.Length, out fdrs);
break;
default:
throw new Exception("Never get here.");
}
mdata.AddNumericColumn(mdata.ColumnNames[col] + " Significance A", "", pvals);
mdata.AddCategoryColumn(mdata.ColumnNames[col] + " A significant", "", fdr);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] rcols = param.GetParam <int[]>("Ratio columns").Value;
int[] icols = param.GetParam <int[]>("Intensity columns").Value;
if (rcols.Length == 0)
{
processInfo.ErrString = "Please specify some ratio columns.";
return;
}
if (rcols.Length != icols.Length)
{
processInfo.ErrString = "The number of ratio and intensity columns have to be equal.";
return;
}
int truncIndex = param.GetParam <int>("Use for truncation").Value;
TestTruncation truncation = truncIndex == 0
? TestTruncation.Pvalue
: (truncIndex == 1 ? TestTruncation.BenjaminiHochberg : TestTruncation.PermutationBased);
double threshold = param.GetParam <double>("Threshold value").Value;
int sideInd = param.GetParam <int>("Side").Value;
TestSide side;
switch (sideInd)
{
case 0:
side = TestSide.Both;
break;
case 1:
side = TestSide.Left;
break;
case 2:
side = TestSide.Right;
break;
default:
throw new Exception("Never get here.");
}
for (int i = 0; i < rcols.Length; i++)
{
BaseVector r = mdata.Values.GetColumn(rcols[i]);
BaseVector intens = icols[i] < mdata.ColumnCount
? mdata.Values.GetColumn(icols[i])
: new DoubleArrayVector(mdata.NumericColumns[icols[i] - mdata.ColumnCount]);
double[] pvals = CalcSignificanceB(r, intens, side);
string[][] fdr;
switch (truncation)
{
case TestTruncation.Pvalue:
fdr = PerseusPluginUtils.CalcPvalueSignificance(pvals, threshold);
break;
case TestTruncation.BenjaminiHochberg:
fdr = PerseusPluginUtils.CalcBenjaminiHochbergFdr(pvals, threshold, out double[] fdrs);
break;
default:
throw new Exception("Never get here.");
}
mdata.AddNumericColumn(mdata.ColumnNames[rcols[i]] + " Significance B", "", pvals);
mdata.AddCategoryColumn(mdata.ColumnNames[rcols[i]] + " B significant", "", fdr);
}
}
private static void TwoSamplesTest1(int[] colInd1, int[] colInd2, TestTruncation truncation, double threshold,
TwoSampleTest test, OneSampleTest test1, TestSide side, bool log, IMatrixData data, double s0, int nrand,
List <int[]> colIndsPreserve1, List <int[]> colIndsPreserve2, string suffix, string mainSuffix, bool filterValidValues,
int minNumValidValues, int minNumValidValuesMode, int minPercValidValues, int minPercValidValuesMode,
out double[] pvalsS0, bool randomized, bool addQval, IList <string> plotNames, IList <string> pvals1Name,
IList <string> fdrs1Name, IList <string> diffs1Name, IList <string> statCol1Name, IList <string> significant1Name,
IList <double[]> pvals1, IList <double[]> fdrs1, IList <double[]> diffs1, IList <double[]> statCol1,
IList <string[][]> significant1, int ind, bool paired)
{
double[] pvals = new double[data.RowCount];
pvalsS0 = new double[data.RowCount];
double[] diffs = new double[data.RowCount];
double[] statCol = new double[data.RowCount];
List <int> validRows = new List <int>();
for (int i = 0; i < data.RowCount; i++)
{
bool valid = IsValidRow(filterValidValues, minNumValidValues, minNumValidValuesMode, minPercValidValues,
minPercValidValuesMode, i, colInd1, colInd2, data);
if (valid)
{
validRows.Add(i);
}
double[] vals1;
double[] vals2;
if (!paired)
{
vals1 = GetValues(i, colInd1, data, true);
vals2 = GetValues(i, colInd2, data, true);
}
else
{
vals1 = GetValues(i, colInd1, data, false);
vals2 = GetValues(i, colInd2, data, false);
}
if (valid)
{
CalcTest(test, test1, paired, vals1, vals2, s0, side, out diffs[i], out pvals[i], out pvalsS0[i], out statCol[i]);
}
else
{
pvals[i] = double.NaN;
pvalsS0[i] = double.NaN;
statCol[i] = double.NaN;
}
}
if (!randomized)
{
string[][] significant;
double[] fdrs = null;
switch (truncation)
{
case TestTruncation.Pvalue:
significant = PerseusPluginUtils.CalcPvalueSignificance(pvals, threshold);
break;
case TestTruncation.BenjaminiHochberg:
significant = PerseusPluginUtils.CalcBenjaminiHochbergFdr(pvals, threshold, out fdrs);
break;
case TestTruncation.PermutationBased:
double[] fdrsX;
string[][] significantX = CalcPermutationBasedFdr(ArrayUtils.SubArray(pvalsS0, validRows), nrand, data, test,
test1, side, colInd1, colInd2, s0, threshold, colIndsPreserve1, colIndsPreserve2, validRows, out fdrsX, paired);
significant = new string[data.RowCount][];
fdrs = new double[data.RowCount];
for (int i = 0; i < significant.Length; i++)
{
significant[i] = new string[0];
fdrs[i] = 1;
}
for (int i = 0; i < validRows.Count; i++)
{
significant[validRows[i]] = significantX[i];
fdrs[validRows[i]] = fdrsX[i];
}
break;
default: throw new Exception("Never get here.");
}
string x = test.Name + " p-value";
if (log)
{
x = "-Log " + x;
for (int i = 0; i < pvals.Length; i++)
{
pvals[i] = -Math.Log10(Math.Max(pvals[i], double.Epsilon));
}
}
plotNames[ind] = suffix + mainSuffix;
pvals1Name[ind] = x + " " + suffix + mainSuffix;
fdrs1Name[ind] = test.Name + " q-value " + suffix + mainSuffix;
diffs1Name[ind] = test.Name + " Difference " + suffix + mainSuffix;
statCol1Name[ind] = test.Name + " Test statistic " + suffix + mainSuffix;
significant1Name[ind] = test.Name + " Significant " + suffix + mainSuffix;
pvals1[ind] = pvals;
if (addQval)
{
fdrs1[ind] = fdrs;
}
diffs1[ind] = diffs;
statCol1[ind] = statCol;
significant1[ind] = significant;
}
}
private static void PerformSingleTest(string firstGroup, string secondGroup, SecondGroupMode secondGroupMode,
IMatrixData mdata, int groupInd, ref string err, bool log, double threshold, TestTruncation truncation, int nrand,
int preserveGroupInd, TwoSampleTest test, OneSampleTest test1, TestSide side, double s0, bool filterValidValues,
int minNumValidValues, int minNumValidValuesMode, int minPercValidValues, int minPercValidValuesMode,
out double[] pvalsS0, IDictionary <int, int> indMap, bool qval, string[] plotNames, string[] pvals1Name,
string[] fdrs1Name, string[] diffs1Name, string[] statCol1Name, string[] significant1Name, double[][] pvals1,
double[][] fdrs1, double[][] diffs1, double[][] statCol1, string[][][] significant1, int ind, List <string>[] sigCol,
out string suffix, string mainSuffix, bool paired)
{
bool randomized = indMap != null;
bool addQval = qval && truncation != TestTruncation.Pvalue;
string[][] groupCol = mdata.GetCategoryRowAt(groupInd);
int[] colInds1;
int[] colInds2;
if (secondGroupMode == SecondGroupMode.Complement)
{
int[][] colInds = PerseusPluginUtils.GetMainColIndices(groupCol, new[] { firstGroup });
colInds1 = colInds[0];
colInds2 = ArrayUtils.Complement(colInds1, mdata.ColumnCount);
}
else
{
int[][] colInds = PerseusPluginUtils.GetMainColIndices(groupCol, new[] { firstGroup, secondGroup });
colInds1 = colInds[0];
colInds2 = colInds[1];
}
if (indMap != null)
{
Transform(colInds1, indMap);
Transform(colInds2, indMap);
}
Array.Sort(colInds1);
Array.Sort(colInds2);
suffix = firstGroup;
if (secondGroupMode != SecondGroupMode.Complement)
{
suffix += "_" + secondGroup;
}
if (paired && colInds1.Length != colInds2.Length)
{
err = "Group sizes have to be equal for paired test.";
pvalsS0 = null;
return;
}
List <int[]> colIndsPreserve1 = null;
List <int[]> colIndsPreserve2 = null;
if (truncation == TestTruncation.PermutationBased)
{
if (preserveGroupInd >= 0)
{
if (paired)
{
err = "Preserved subgroups are not supported for paired tests.";
pvalsS0 = null;
return;
}
string[][] preserveGroupCol = mdata.GetCategoryRowAt(preserveGroupInd);
string[] allGroupsPreserve = ArrayUtils.UniqueValuesPreserveOrder(ArrayUtils.Concat(preserveGroupCol));
int[][] colIndsPreserve = PerseusPluginUtils.GetMainColIndices(preserveGroupCol, allGroupsPreserve);
int[] allInds = ArrayUtils.Concat(colIndsPreserve);
int[] allIndsUnique = ArrayUtils.UniqueValues(allInds);
if (allInds.Length != allIndsUnique.Length)
{
err = "The grouping for randomizations is not unique";
pvalsS0 = null;
return;
}
if (allInds.Length != colInds1.Length + colInds2.Length)
{
err = "The grouping for randomizations is not valid because it does not cover all samples.";
pvalsS0 = null;
return;
}
colIndsPreserve1 = new List <int[]>();
colIndsPreserve2 = new List <int[]>();
foreach (int[] inds in colIndsPreserve)
{
int index = DetermineGroup(colInds1, colInds2, inds);
if (index == 0)
{
err = "The grouping for randomizations is not hierarchical with respect to the main grouping.";
pvalsS0 = null;
return;
}
switch (index)
{
case 1:
colIndsPreserve1.Add(inds);
break;
case 2:
colIndsPreserve2.Add(inds);
break;
}
}
}
}
TwoSamplesTest1(colInds1, colInds2, truncation, threshold, test, test1, side, log, mdata, s0, nrand,
colIndsPreserve1, colIndsPreserve2, suffix, mainSuffix, filterValidValues, minNumValidValues, minNumValidValuesMode,
minPercValidValues, minPercValidValuesMode, out pvalsS0, randomized, addQval, plotNames, pvals1Name, fdrs1Name,
diffs1Name, statCol1Name, significant1Name, pvals1, fdrs1, diffs1, statCol1, significant1, ind, paired);
if (significant1 != null)
{
for (int i = 0; i < significant1[ind].Length; i++)
{
if (significant1[ind][i].Length > 0)
{
sigCol[i].Add(suffix);
}
}
}
}
public static double[] Process(IList <int> firstGroupInds, IList <int> secondGroupInds,
SecondGroupMode secondGroupMode, IList <string> groupNames, IMatrixData mdata, int groupInd, ProcessInfo processInfo,
bool log, double threshold, TestTruncation truncation, int preserveGroupInd, int nrand, TwoSampleTest test,
OneSampleTest test1, TestSide side, double s0, bool filterValidValues, int minNumValidValues,
int minNumValidValuesMode, int minPercValidValues, int minPercValidValuesMode, bool calcCombinedScore,
CombinedScoreMode combinedScoreMode, IDictionary <int, int> indMap, bool qval, string[] plotNames,
string[] pvals1Name, string[] fdrs1Name, string[] diffs1Name, string[] statCol1Name, string[] significant1Name,
double[][] pvals1, double[][] fdrs1, double[][] diffs1, double[][] statCol1, string[][][] significant1,
out string[][] sig, out string[] testNames, string mainSuffix, bool paired)
{
int ntests = firstGroupInds.Count;
double[][] pvalsS0 = new double[ntests][];
List <string>[] sigCol = null;
if (significant1 != null)
{
sigCol = new List <string> [mdata.RowCount];
for (int i = 0; i < sigCol.Length; i++)
{
sigCol[i] = new List <string>();
}
}
testNames = new string[ntests];
for (int itest = 0; itest < ntests; itest++)
{
string firstGroup = groupNames[firstGroupInds[itest]];
string secondGroup = secondGroupMode == SecondGroupMode.Complement ? null : groupNames[secondGroupInds[itest]];
string err = null;
PerformSingleTest(firstGroup, secondGroup, secondGroupMode, mdata, groupInd, ref err, log, threshold, truncation,
nrand, preserveGroupInd, test, test1, side, s0, filterValidValues, minNumValidValues, minNumValidValuesMode,
minPercValidValues, minPercValidValuesMode, out pvalsS0[itest], indMap, qval, plotNames, pvals1Name, fdrs1Name,
diffs1Name, statCol1Name, significant1Name, pvals1, fdrs1, diffs1, statCol1, significant1, itest, sigCol,
out testNames[itest], mainSuffix, paired);
if (!string.IsNullOrEmpty(err))
{
processInfo.ErrString = err;
sig = null;
return(null);
}
}
sig = null;
if (significant1 != null)
{
sig = new string[sigCol.Length][];
for (int i = 0; i < sig.Length; i++)
{
sigCol[i].Sort();
sig[i] = sigCol[i].ToArray();
}
}
if (calcCombinedScore)
{
double[] combinedPvalsS0 = new double[mdata.RowCount];
for (int i = 0; i < mdata.RowCount; i++)
{
double[] x = ExtractRow(pvalsS0, i);
switch (combinedScoreMode)
{
case CombinedScoreMode.Product:
combinedPvalsS0[i] = ArrayUtils.Product(x);
break;
case CombinedScoreMode.ProductOfSignificant: throw new NotImplementedException();
case CombinedScoreMode.Best:
combinedPvalsS0[i] = ArrayUtils.Min(x);
break;
default: throw new Exception("Never get here");
}
}
return(combinedPvalsS0);
}
return(null);
}
public static void ReadParams(bool isVolcano, IMatrixData mdata, Parameters param, ProcessInfo processInfo,
out int groupRowInd, out int[] firstGroupInds, out int[] secondGroupInds, out SecondGroupMode secondGroupMode,
out string[] groupNames, out bool logPval, out double threshold, out TestTruncation truncation,
out int preserveGroupInd, out int nrand, out TwoSampleTest test, out TestSide side, out double s0,
out bool filterValidValues, out int minNumValidValues, out int minNumValidValuesMode, out int minPercValidValues,
out int minPercValidValuesMode, out bool calcCombinedScore, out CombinedScoreMode combinedScoreMode,
out bool combinedScoreQvalue, out bool qval, out string suffix, out bool paired, out OneSampleTest test1,
TwoSampleTest[] allTests)
{
ParameterWithSubParams <int> groupingParam = param.GetParamWithSubParams <int>("Grouping");
groupRowInd = groupingParam.Value;
Parameters subparam = groupingParam.GetSubParameters();
firstGroupInds = subparam.GetParam <int[]>("First group (right)").Value;
ParameterWithSubParams <int> p = subparam.GetParamWithSubParams <int>("Second groups mode");
int modeInd = p.Value;
paired = false;
switch (modeInd)
{
case 0:
secondGroupMode = SecondGroupMode.SpecifiyAll;
secondGroupInds = p.GetSubParameters().GetParam <int[]>("Second group (left)").Value;
paired = p.GetSubParameters().GetParam <bool>("Paired").Value;
break;
case 1:
secondGroupMode = SecondGroupMode.SingleControl;
int ind = p.GetSubParameters().GetParam <int>("Second group (left)").Value;
secondGroupInds = new int[firstGroupInds.Length];
for (int i = 0; i < secondGroupInds.Length; i++)
{
secondGroupInds[i] = ind;
}
paired = p.GetSubParameters().GetParam <bool>("Paired").Value;
break;
case 2:
secondGroupMode = SecondGroupMode.Complement;
secondGroupInds = new int[0];
break;
default: throw new Exception("Never get here.");
}
groupNames = null;
logPval = false;
suffix = "";
threshold = 0;
truncation = TestTruncation.PermutationBased;
preserveGroupInd = 0;
nrand = 0;
test = null;
side = TestSide.Both;
s0 = 0;
filterValidValues = false;
minNumValidValues = 0;
minNumValidValuesMode = 0;
minPercValidValues = 0;
minPercValidValuesMode = 0;
calcCombinedScore = false;
combinedScoreMode = CombinedScoreMode.Product;
combinedScoreQvalue = false;
test1 = null;
qval = false;
if (secondGroupMode != SecondGroupMode.Complement && firstGroupInds.Length != secondGroupInds.Length)
{
processInfo.ErrString = "Please specify the same number of groups in the 'First group' and 'Second group' boxes.";
return;
}
if (firstGroupInds.Length == 0)
{
processInfo.ErrString = "Please specify some groups.";
return;
}
if (secondGroupMode != SecondGroupMode.Complement)
{
for (int i = 0; i < firstGroupInds.Length; i++)
{
if (firstGroupInds[i] == secondGroupInds[i])
{
processInfo.ErrString = "Groups to be compared in the test cannot be equal.";
return;
}
}
}
groupNames = ArrayUtils.UniqueValuesPreserveOrder(ArrayUtils.Concat(mdata.GetCategoryRowAt(groupRowInd)));
logPval = param.GetParam <bool>("-Log10 p-value").Value;
suffix = param.GetParam <string>("Suffix").Value;
ParameterWithSubParams <int> truncParam = param.GetParamWithSubParams <int>("Use for truncation");
if (!isVolcano)
{
int truncIndex = truncParam.Value;
truncation = truncIndex == 0
? TestTruncation.Pvalue
: (truncIndex == 1 ? TestTruncation.BenjaminiHochberg : TestTruncation.PermutationBased);
}
Parameters truncSubParams = truncParam.GetSubParameters();
threshold = truncation == TestTruncation.Pvalue
? truncParam.GetSubParameters().GetParam <double>("Threshold p-value").Value
: truncParam.GetSubParameters().GetParam <double>("FDR").Value;
ParameterWithSubParams <int> testParam = param.GetParamWithSubParams <int>("Test");
int testInd = testParam.Value;
ParameterWithSubParams <bool> validValParam = param.GetParamWithSubParams <bool>("Valid value filter");
filterValidValues = validValParam.Value;
minNumValidValues = 0;
minNumValidValuesMode = 0;
minPercValidValues = 0;
minPercValidValuesMode = 0;
if (filterValidValues)
{
Parameters px = validValParam.GetSubParameters();
minNumValidValues = px.GetParam <int>("Min. number of valid values").Value;
minNumValidValuesMode = px.GetParam <int>("Min. number mode").Value;
minPercValidValues = px.GetParam <int>("Min. percentage of valid values").Value;
minPercValidValuesMode = px.GetParam <int>("Min. percentage mode").Value;
}
int sideInd = 0;
test = allTests[testInd];
Parameters testSubParams = testParam.GetSubParameters();
if (test.HasSides)
{
sideInd = testSubParams.GetParam <int>("Side").Value;
}
switch (sideInd)
{
case 0:
side = TestSide.Both;
break;
case 1:
side = TestSide.Left;
break;
case 2:
side = TestSide.Right;
break;
default: throw new Exception("Never get here.");
}
s0 = 0;
if (test.HasS0)
{
s0 = testSubParams.GetParam <double>("S0").Value;
}
if (paired)
{
test1 = test.GetOneSampleTest();
}
ParameterWithSubParams <bool> combinedScoreParam = param.GetParamWithSubParams <bool>("Calculate combined score");
calcCombinedScore = combinedScoreParam.Value;
if (firstGroupInds.Length < 2 || truncation != TestTruncation.PermutationBased)
{
calcCombinedScore = false;
}
combinedScoreMode = CombinedScoreMode.Product;
if (calcCombinedScore)
{
Parameters px = combinedScoreParam.GetSubParameters();
combinedScoreQvalue = px.GetParam <bool>("Combined q-value").Value;
int combinedScoreBestInd = px.GetParam <int>("Mode").Value;
switch (combinedScoreBestInd)
{
case 0:
combinedScoreMode = CombinedScoreMode.Product;
break;
case 1:
combinedScoreMode = CombinedScoreMode.Best;
break;
default: throw new Exception("Never get here.");
}
}
if (truncation != TestTruncation.Pvalue)
{
qval = truncSubParams.GetParam <bool>("Report q-value").Value;
}
nrand = -1;
if (truncation == TestTruncation.PermutationBased)
{
nrand = truncSubParams.GetParam <int>("Number of randomizations").Value;
preserveGroupInd = truncSubParams.GetParam <int>("Preserve grouping in randomizations").Value - 1;
}
if (preserveGroupInd >= 0 && calcCombinedScore)
{
processInfo.ErrString = "Combination of preserved subgroups and combined score is not yet supported.";
}
if (paired && combinedScoreQvalue)
{
processInfo.ErrString = "Combined q-value is not supported for paired tests.";
}
}
