public IMatrixData CombineData(IMatrixData matrixData1, IMatrixData matrixData2, Parameters parameters,
ProcessInfo processInfo)
{
bool indicator = parameters.GetBoolParam("Indicator").Value;
int otherCol = parameters.GetSingleChoiceParam("Matching column 2").Value;
Average avExpression = GetAveraging(parameters.GetSingleChoiceParam("Combine expression values").Value);
Average avNumerical = GetAveraging(parameters.GetSingleChoiceParam("Combine numerical values").Value);
string[] q = matrixData2.StringColumns[otherCol];
string[][] w = new string[q.Length][];
for (int i = 0; i < q.Length; i++){
string r = q[i].Trim();
w[i] = r.Length == 0 ? new string[0] : r.Split(';');
w[i] = ArrayUtils.UniqueValues(w[i]);
}
Dictionary<string, List<int>> id2Cols = new Dictionary<string, List<int>>();
for (int i = 0; i < w.Length; i++){
foreach (string s in w[i]){
if (!id2Cols.ContainsKey(s)){
id2Cols.Add(s, new List<int>());
}
id2Cols[s].Add(i);
}
}
int pgCol = parameters.GetSingleChoiceParam("Matching column 1").Value;
string[] d = matrixData1.StringColumns[pgCol];
string[][] x = new string[d.Length][];
for (int i = 0; i < d.Length; i++){
string r = d[i].Trim();
x[i] = r.Length == 0 ? new string[0] : r.Split(';');
x[i] = ArrayUtils.UniqueValues(x[i]);
}
int[][] indexMap = new int[x.Length][];
string[][] indicatorCol = new string[x.Length][];
for (int i = 0; i < indexMap.Length; i++){
List<int> qwer = new List<int>();
foreach (string s in x[i]){
if (id2Cols.ContainsKey(s)){
List<int> en = id2Cols[s];
qwer.AddRange(en);
}
}
indexMap[i] = qwer.ToArray();
indexMap[i] = ArrayUtils.UniqueValues(indexMap[i]);
indicatorCol[i] = indexMap[i].Length > 0 ? new[]{"+"} : new string[0];
}
IMatrixData result = matrixData1.Copy();
SetAnnotationRows(result, matrixData1, matrixData2);
if (indicator){
result.AddCategoryColumn(matrixData2.Name, "", indicatorCol);
}
{
int[] exCols = parameters.GetMultiChoiceParam("Expression columns").Value;
float[,] newExColumns = new float[matrixData1.RowCount, exCols.Length];
float[,] newQuality = new float[matrixData1.RowCount, exCols.Length];
bool[,] newIsImputed = new bool[matrixData1.RowCount, exCols.Length];
string[] newExColNames = new string[exCols.Length];
float[,] oldEx = matrixData2.ExpressionValues;
float[,] oldQual = matrixData2.QualityValues;
bool[,] oldImp = matrixData2.IsImputed;
for (int i = 0; i < exCols.Length; i++) {
newExColNames[i] = matrixData2.ExpressionColumnNames[exCols[i]];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<double> values = new List<double>();
List<double> qual = new List<double>();
List<bool> imp = new List<bool>();
foreach (int ind in inds) {
double v = oldEx[ind, exCols[i]];
if (!double.IsNaN(v) && !double.IsInfinity(v)){
values.Add(v);
double qx = oldQual[ind, exCols[i]];
if (!double.IsNaN(qx) && !double.IsInfinity(qx)){
qual.Add(qx);
}
bool isi = oldImp[ind, exCols[i]];
imp.Add(isi);
}
}
newExColumns[j, i] = values.Count == 0 ? float.NaN : (float)avExpression(values.ToArray());
newQuality[j, i] = qual.Count == 0 ? float.NaN : (float)avExpression(qual.ToArray());
newIsImputed[j, i] = imp.Count != 0 && AvImp(imp.ToArray());
}
}
MakeNewNames(newExColNames, result.ExpressionColumnNames);
AddExpressionColumns(result, newExColNames, newExColumns, newQuality, newIsImputed);
}
{
int[] numCols = parameters.GetMultiChoiceParam("Numerical columns").Value;
double[][] newNumericalColumns = new double[numCols.Length][];
string[] newNumColNames = new string[numCols.Length];
for (int i = 0; i < numCols.Length; i++){
double[] oldCol = matrixData2.NumericColumns[numCols[i]];
newNumColNames[i] = matrixData2.NumericColumnNames[numCols[i]];
newNumericalColumns[i] = new double[matrixData1.RowCount];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<double> values = new List<double>();
foreach (int ind in inds){
double v = oldCol[ind];
if (!double.IsNaN(v)){
values.Add(v);
}
}
newNumericalColumns[i][j] = values.Count == 0 ? double.NaN : avNumerical(values.ToArray());
}
}
for (int i = 0; i < numCols.Length; i++){
result.AddNumericColumn(newNumColNames[i], "", newNumericalColumns[i]);
}
}
{
int[] catCols = parameters.GetMultiChoiceParam("Categorical columns").Value;
string[][][] newCatColumns = new string[catCols.Length][][];
string[] newCatColNames = new string[catCols.Length];
for (int i = 0; i < catCols.Length; i++){
string[][] oldCol = matrixData2.CategoryColumns[catCols[i]];
newCatColNames[i] = matrixData2.CategoryColumnNames[catCols[i]];
newCatColumns[i] = new string[matrixData1.RowCount][];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<string[]> values = new List<string[]>();
foreach (int ind in inds){
string[] v = oldCol[ind];
if (v.Length > 0){
values.Add(v);
}
}
newCatColumns[i][j] = values.Count == 0
? new string[0] : ArrayUtils.UniqueValues(ArrayUtils.Concat(values.ToArray()));
}
}
for (int i = 0; i < catCols.Length; i++){
result.AddCategoryColumn(newCatColNames[i], "", newCatColumns[i]);
}
}
{
int[] stringCols = parameters.GetMultiChoiceParam("String columns").Value;
string[][] newStringColumns = new string[stringCols.Length][];
string[] newStringColNames = new string[stringCols.Length];
for (int i = 0; i < stringCols.Length; i++){
string[] oldCol = matrixData2.StringColumns[stringCols[i]];
newStringColNames[i] = matrixData2.StringColumnNames[stringCols[i]];
newStringColumns[i] = new string[matrixData1.RowCount];
for (int j = 0; j < matrixData1.RowCount; j++){
int[] inds = indexMap[j];
List<string> values = new List<string>();
foreach (int ind in inds){
string v = oldCol[ind];
if (v.Length > 0){
values.Add(v);
}
}
newStringColumns[i][j] = values.Count == 0 ? "" : StringUtils.Concat(";", values.ToArray());
}
}
for (int i = 0; i < stringCols.Length; i++){
result.AddStringColumn(newStringColNames[i], "", newStringColumns[i]);
}
}
result.Origin = "Combination";
return result;
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] cols = param.GetMultiChoiceParam("Columns").Value;
int truncIndex = param.GetSingleChoiceParam("Use for truncation").Value;
TestTruncation truncation = truncIndex == 0
? TestTruncation.Pvalue : (truncIndex == 1 ? TestTruncation.BenjaminiHochberg : TestTruncation.PermutationBased);
double threshold = param.GetDoubleParam("Threshold value").Value;
int sideInd = param.GetSingleChoiceParam("Side").Value;
TestSide side;
switch (sideInd){
case 0:
side = TestSide.Both;
break;
case 1:
side = TestSide.Left;
break;
case 2:
side = TestSide.Right;
break;
default:
throw new Exception("Never get here.");
}
foreach (int col in cols){
float[] r = mdata.GetExpressionColumn(col);
double[] pvals = CalcSignificanceA(r, side);
string[][] fdr;
switch (truncation){
case TestTruncation.Pvalue:
fdr = PerseusPluginUtils.CalcPvalueSignificance(pvals, threshold);
break;
case TestTruncation.BenjaminiHochberg:
fdr = PerseusPluginUtils.CalcBenjaminiHochbergFdr(pvals, threshold);
break;
default:
throw new Exception("Never get here.");
}
mdata.AddNumericColumn(mdata.ExpressionColumnNames[col] + " Significance A", "", pvals);
mdata.AddCategoryColumn(mdata.ExpressionColumnNames[col] + " A significant", "", fdr);
}
}
public void ProcessData(IMatrixData data, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
bool falseAreIndicated = param.GetSingleChoiceParam("Indicated are").Value == 0;
int catCol = param.GetSingleChoiceParam("In column").Value;
string word = param.GetStringParam("Indicator").Value;
int[] scoreColumns = param.GetMultiChoiceParam("Scores").Value;
if (scoreColumns.Length == 0){
processInfo.ErrString = "Please specify at least one column with scores.";
return;
}
bool largeIsGood = param.GetBoolParam("Large values are good").Value;
int[] showColumns = param.GetMultiChoiceParam("Display quantity").Value;
if (showColumns.Length == 0){
processInfo.ErrString = "Please select at least one quantity to display";
return;
}
bool[] indCol = GetIndicatorColumn(falseAreIndicated, catCol, word, data);
List<string> expColNames = new List<string>();
List<float[]> expCols = new List<float[]>();
foreach (int scoreColumn in scoreColumns){
double[] vals = scoreColumn < data.NumericColumnCount
? data.NumericColumns[scoreColumn]
: ArrayUtils.ToDoubles(data.GetExpressionColumn(scoreColumn - data.NumericColumnCount));
string name = scoreColumn < data.NumericColumnCount
? data.NumericColumnNames[scoreColumn] : data.ExpressionColumnNames[scoreColumn - data.NumericColumnCount];
int[] order = GetOrder(vals, largeIsGood);
CalcCurve(ArrayUtils.SubArray(indCol, order), showColumns, name, expCols, expColNames);
}
float[,] expData = ToMatrix(expCols);
data.SetData(data.Name, expColNames, expData, new List<string>(), new List<string[]>(), new List<string>(),
new List<string[][]>(), new List<string>(), new List<double[]>(), new List<string>(), new List<double[][]>());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] cols = param.GetMultiChoiceParam("Columns").Value;
HashSet<int> w = ArrayUtils.ToHashSet(param.GetMultiChoiceParam("Calculate").Value);
bool[] include = new bool[SummaryStatisticsRows.procs.Length];
double[][] rowws = new double[SummaryStatisticsRows.procs.Length][];
for (int i = 0; i < include.Length; i++){
include[i] = w.Contains(i);
if (include[i]){
rowws[i] = new double[cols.Length];
}
}
for (int i = 0; i < cols.Length; i++){
double[] vals = GetColumn(cols[i], mdata);
for (int j = 0; j < include.Length; j++){
if (include[j]){
rowws[j][i] = SummaryStatisticsRows.procs[j].Item2(vals);
}
}
}
List<double[]> ex = new List<double[]>();
List<string> names = new List<string>();
for (int i = 0; i < include.Length; i++){
if (include[i]){
ex.Add(rowws[i]);
names.Add(SummaryStatisticsRows.procs[i].Item1);
}
}
float[,] exVals = GetExVals(ex);
string[] colNames = GetColNames(mdata, cols);
mdata.SetData("Summary", new List<string>(names.ToArray()), exVals, new List<string>(new[]{"Columns"}),
new List<string[]>(new[]{colNames}), mdata.CategoryRowNames,
TransformCategories(mdata, cols, mdata.ExpressionColumnCount), mdata.NumericRowNames,
TransformNumeric(mdata.NumericRows, cols, mdata.ExpressionColumnCount), new List<string>(), new List<double[][]>());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int numQuantiles = param.GetIntParam("Number of quantiles").Value;
int[] colInds = param.GetMultiChoiceParam("Columns").Value;
foreach (int colInd in colInds){
float[] vals = mdata.GetExpressionColumn(colInd);
List<int> v = new List<int>();
for (int i = 0; i < vals.Length; i++){
if (!float.IsNaN(vals[i])){
v.Add(i);
}
}
int[] o = v.ToArray();
vals = ArrayUtils.SubArray(vals, o);
int[] q = ArrayUtils.Order(vals);
o = ArrayUtils.SubArray(o, q);
string[][] catCol = new string[mdata.RowCount][];
for (int i = 0; i < catCol.Length; i++){
catCol[i] = new[]{"missing"};
}
for (int i = 0; i < o.Length; i++){
int catVal = (i*numQuantiles)/o.Length + 1;
catCol[o[i]] = new[]{"Q" + catVal};
}
string name = mdata.ExpressionColumnNames[colInd] + "_q";
string desc = "The column " + mdata.ExpressionColumnNames[colInd] + " has been divided into " + numQuantiles +
" quantiles.";
mdata.AddCategoryColumn(name, desc, catCol);
}
}
public void ProcessData(IMatrixData data, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] exColInds = param.GetMultiChoiceParam("Expression columns").Value;
int[] numColInds = param.GetMultiChoiceParam("Numerical columns").Value;
int[] multiNumColInds = param.GetMultiChoiceParam("Multi-numerical columns").Value;
int[] catColInds = param.GetMultiChoiceParam("Categorical columns").Value;
int[] textColInds = param.GetMultiChoiceParam("Text columns").Value;
data.ExtractExpressionColumns(exColInds);
data.NumericColumns = ArrayUtils.SubList(data.NumericColumns, numColInds);
data.NumericColumnNames = ArrayUtils.SubList(data.NumericColumnNames, numColInds);
data.NumericColumnDescriptions = ArrayUtils.SubList(data.NumericColumnDescriptions, numColInds);
data.MultiNumericColumns = ArrayUtils.SubList(data.MultiNumericColumns, multiNumColInds);
data.MultiNumericColumnNames = ArrayUtils.SubList(data.MultiNumericColumnNames, multiNumColInds);
data.MultiNumericColumnDescriptions = ArrayUtils.SubList(data.MultiNumericColumnDescriptions, multiNumColInds);
data.CategoryColumns = PerseusPluginUtils.GetCategoryColumns(data, catColInds);
data.CategoryColumnNames = ArrayUtils.SubList(data.CategoryColumnNames, catColInds);
data.CategoryColumnDescriptions = ArrayUtils.SubList(data.CategoryColumnDescriptions, catColInds);
data.StringColumns = ArrayUtils.SubList(data.StringColumns, textColInds);
data.StringColumnNames = ArrayUtils.SubList(data.StringColumnNames, textColInds);
data.StringColumnDescriptions = ArrayUtils.SubList(data.StringColumnDescriptions, textColInds);
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] stringCols = param1.GetMultiChoiceParam("String columns").Value;
if (stringCols.Length == 0){
processInfo.ErrString = "Please select some columns.";
return;
}
foreach (string[] col in stringCols.Select(stringCol => mdata.StringColumns[stringCol])){
for (int i = 0; i < col.Length; i++){
string q = col[i];
if (q.Length == 0){
continue;
}
string[] w = q.Split(';');
w = ArrayUtils.UniqueValues(w);
col[i] = StringUtils.Concat(";", w);
}
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] colIndx = param.GetMultiChoiceParam("x").Value;
int[] colIndy = param.GetMultiChoiceParam("y").Value;
if (colIndx.Length == 0){
processInfo.ErrString = "Please select some columns";
return;
}
if (colIndx.Length != colIndy.Length){
processInfo.ErrString = "Please select the same number of columns in the boxes for the first and second columns.";
return;
}
int typeInd = param.GetSingleChoiceParam("Distribution type").Value;
int points = param.GetIntParam("Number of points").Value;
for (int k = 0; k < colIndx.Length; k++){
float[] xvals = GetColumn(mdata, colIndx[k]);
float[] yvals = GetColumn(mdata, colIndy[k]);
float[] xvals1;
float[] yvals1;
NumUtils.GetValidPairs(xvals, yvals, out xvals1, out yvals1);
double xmin;
double xmax;
double ymin;
double ymax;
DensityEstimation.CalcRanges(xvals1, yvals1, out xmin, out xmax, out ymin, out ymax);
float[,] values = DensityEstimation.GetValuesOnGrid(xvals1, xmin, (xmax - xmin)/points, points, yvals1, ymin,
(ymax - ymin)/points, points);
if (typeInd == 1 || typeInd == 3){
MakeConditional1(values);
}
if (typeInd == 2 || typeInd == 3){
MakeConditional2(values);
}
DensityEstimation.DivideByMaximum(values);
double[] xmat = new double[points];
for (int i = 0; i < points; i++){
xmat[i] = xmin + i*(xmax - xmin)/points;
}
double[] ymat = new double[points];
for (int i = 0; i < points; i++){
ymat[i] = ymin + i*(ymax - ymin)/points;
}
float[,] percvalues = CalcExcludedPercentage(values);
double[] dvals = new double[xvals.Length];
double[] pvals = new double[xvals.Length];
for (int i = 0; i < dvals.Length; i++){
double xx = xvals[i];
double yy = yvals[i];
if (!double.IsNaN(xx) && !double.IsNaN(yy)){
int xind = ArrayUtils.ClosestIndex(xmat, xx);
int yind = ArrayUtils.ClosestIndex(ymat, yy);
dvals[i] = values[xind, yind];
pvals[i] = percvalues[xind, yind];
} else{
dvals[i] = double.NaN;
pvals[i] = double.NaN;
}
}
string xname = GetColumnName(mdata, colIndx[k]);
string yname = GetColumnName(mdata, colIndy[k]);
mdata.AddNumericColumn("Density_" + xname + "_" + yname,
"Density of data points in the plane spanned by the columns " + xname + " and " + yname + ".", dvals);
mdata.AddNumericColumn("Excluded fraction_" + xname + "_" + yname,
"Percentage of points with a point density smaller than at this point in the plane spanned by the columns " + xname +
" and " + yname + ".", pvals);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
SingleChoiceWithSubParams xp = param.GetSingleChoiceWithSubParams("Expression column selection");
bool groups = xp.Value == 2;
string[] groupNames = null;
int[][] colIndsGroups = null;
if (groups){
int groupRowInd = xp.GetSubParameters().GetSingleChoiceParam("Group").Value;
string[][] groupCol = mdata.GetCategoryRowAt(groupRowInd);
groupNames = ArrayUtils.UniqueValuesPreserveOrder(groupCol);
colIndsGroups = PerseusPluginUtils.GetExpressionColIndices(groupCol, groupNames);
}
int[] useCols = xp.Value == 1
? xp.GetSubParameters().GetMultiChoiceParam("Columns").Value
: ArrayUtils.ConsecutiveInts(mdata.ExpressionColumnCount);
HashSet<int> w = ArrayUtils.ToHashSet(param.GetMultiChoiceParam("Calculate").Value);
bool[] include = new bool[procs.Length];
double[][] columns = new double[procs.Length][];
double[][][] columnsG = null;
if (groups){
columnsG = new double[procs.Length][][];
for (int i = 0; i < columnsG.Length; i++){
columnsG[i] = new double[groupNames.Length][];
}
}
for (int i = 0; i < include.Length; i++){
include[i] = w.Contains(i);
if (include[i]){
columns[i] = new double[mdata.RowCount];
if (groups){
for (int j = 0; j < groupNames.Length; j++){
columnsG[i][j] = new double[mdata.RowCount];
}
}
}
}
for (int i = 0; i < mdata.RowCount; i++){
List<double> v = new List<double>();
foreach (int j in useCols){
double x = mdata[i, j];
if (!double.IsNaN(x) && !double.IsInfinity(x)){
v.Add(x);
}
}
for (int j = 0; j < include.Length; j++){
if (include[j]){
columns[j][i] = procs[j].Item2(v);
}
}
if (groups){
List<double>[] vg = new List<double>[groupNames.Length];
for (int j = 0; j < colIndsGroups.Length; j++){
vg[j] = new List<double>();
for (int k = 0; k < colIndsGroups[j].Length; k++){
double x = mdata[i, colIndsGroups[j][k]];
if (!double.IsNaN(x) && !double.IsInfinity(x)){
vg[j].Add(x);
}
}
}
for (int j = 0; j < include.Length; j++){
if (include[j]){
for (int k = 0; k < groupNames.Length; k++){
columnsG[j][k][i] = procs[j].Item2(vg[k]);
}
}
}
}
}
for (int i = 0; i < include.Length; i++){
if (include[i]){
mdata.AddNumericColumn(procs[i].Item1, procs[i].Item3, columns[i]);
if (groups){
for (int k = 0; k < groupNames.Length; k++){
mdata.AddNumericColumn(procs[i].Item1 + " " + groupNames[k], procs[i].Item3, columnsG[i][k]);
}
}
}
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] rcols = param.GetMultiChoiceParam("Ratio columns").Value;
int[] icols = param.GetMultiChoiceParam("Intensity columns").Value;
if (rcols.Length == 0){
processInfo.ErrString = "Please specify some ratio columns.";
return;
}
if (rcols.Length != icols.Length){
processInfo.ErrString = "The number of ratio and intensity columns have to be equal.";
return;
}
int truncIndex = param.GetSingleChoiceParam("Use for truncation").Value;
TestTruncation truncation = truncIndex == 0
? TestTruncation.Pvalue : (truncIndex == 1 ? TestTruncation.BenjaminiHochberg : TestTruncation.PermutationBased);
double threshold = param.GetDoubleParam("Threshold value").Value;
int sideInd = param.GetSingleChoiceParam("Side").Value;
TestSide side;
switch (sideInd){
case 0:
side = TestSide.Both;
break;
case 1:
side = TestSide.Left;
break;
case 2:
side = TestSide.Right;
break;
default:
throw new Exception("Never get here.");
}
for (int i = 0; i < rcols.Length; i++){
float[] r = mdata.GetExpressionColumn(rcols[i]);
float[] intens = icols[i] < mdata.ExpressionColumnCount
? mdata.GetExpressionColumn(icols[i])
: ArrayUtils.ToFloats(mdata.NumericColumns[icols[i] - mdata.ExpressionColumnCount]);
double[] pvals = CalcSignificanceB(r, intens, side);
string[][] fdr;
switch (truncation){
case TestTruncation.Pvalue:
fdr = PerseusPluginUtils.CalcPvalueSignificance(pvals, threshold);
break;
case TestTruncation.BenjaminiHochberg:
fdr = PerseusPluginUtils.CalcBenjaminiHochbergFdr(pvals, threshold);
break;
default:
throw new Exception("Never get here.");
}
mdata.AddNumericColumn(mdata.ExpressionColumnNames[rcols[i]] + " Significance B", "", pvals);
mdata.AddCategoryColumn(mdata.ExpressionColumnNames[rcols[i]] + " B significant", "", fdr);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] outputColumns = param.GetMultiChoiceParam("Output").Value;
int proteinIdColumnInd = param.GetSingleChoiceParam("Protein IDs").Value;
string[] proteinIds = mdata.StringColumns[proteinIdColumnInd];
int[] intensityCols = param.GetMultiChoiceParam("Intensities").Value;
if (intensityCols.Length == 0){
processInfo.ErrString = "Please select at least one column containing protein intensities.";
return;
}
// variable to hold all intensity values
List<double[]> columns = new List<double[]>();
string[] sampleNames = new string[intensityCols.Length];
for (int col = 0; col < intensityCols.Length; col++){
double[] values;
if (intensityCols[col] < mdata.ExpressionColumnCount){
values = ArrayUtils.ToDoubles(mdata.GetExpressionColumn(intensityCols[col]));
sampleNames[col] = mdata.ExpressionColumnNames[intensityCols[col]];
} else{
values = mdata.NumericColumns[intensityCols[col] - mdata.ExpressionColumnCount];
sampleNames[col] = mdata.NumericColumnNames[intensityCols[col] - mdata.ExpressionColumnCount];
}
sampleNames[col] = new Regex(@"^(?:(?:LFQ )?[Ii]ntensity )?(.*)$").Match(sampleNames[col]).Groups[1].Value;
columns.Add(values);
}
// average over columns if this option is selected
if (param.GetSingleChoiceWithSubParams("Averaging mode").Value == 3){
double[] column = new double[mdata.RowCount];
for (int row = 0; row < mdata.RowCount; row++){
double[] values = new double[intensityCols.Length];
for (int col = 0; col < intensityCols.Length; col++){
values[col] = columns[col][row];
}
column[row] = ArrayUtils.Median(ExtractValidValues(values, false));
}
// delete the original list of columns
columns = new List<double[]>{column};
sampleNames = new[]{""};
}
// revert logarithm if necessary
if (param.GetBoolWithSubParams("Logarithmized").Value){
double[] logBases = new[]{2, Math.E, 10};
double logBase =
logBases[param.GetBoolWithSubParams("Logarithmized").GetSubParameters().GetSingleChoiceParam("log base").Value];
foreach (double[] t in columns){
for (int row = 0; row < mdata.RowCount; row++){
if (t[row] == 0){
processInfo.ErrString = "Are the columns really logarithmized?\nThey contain zeroes!";
}
t[row] = Math.Pow(logBase, t[row]);
}
}
}
double[] mw = mdata.NumericColumns[param.GetSingleChoiceParam("Molecular masses").Value];
// detect whether the molecular masses are given in Da or kDa
if (ArrayUtils.Median(mw) < 250) // likely kDa
{
for (int i = 0; i < mw.Length; i++){
mw[i] *= 1000;
}
}
double[] detectabilityNormFactor = mw;
if (param.GetBoolWithSubParams("Detectability correction").Value){
detectabilityNormFactor =
mdata.NumericColumns[
param.GetBoolWithSubParams("Detectability correction")
.GetSubParameters()
.GetSingleChoiceParam("Correction factor")
.Value];
}
// the normalization factor needs to be nonzero for all proteins
// check and replace with 1 for all relevant cases
for (int row = 0; row < mdata.RowCount; row++){
if (detectabilityNormFactor[row] == 0 || detectabilityNormFactor[row] == double.NaN){
detectabilityNormFactor[row] = 1;
}
}
// detect the organism
Organism organism = DetectOrganism(proteinIds);
// c value the amount of DNA per cell, see: http://en.wikipedia.org/wiki/C-value
double cValue = (organism.genomeSize*basePairWeight)/avogadro;
// find the histones
int[] histoneRows = FindHistones(proteinIds, organism);
// write a categorical column indicating the histones
string[][] histoneCol = new string[mdata.RowCount][];
for (int row = 0; row < mdata.RowCount; row++){
histoneCol[row] = (ArrayUtils.Contains(histoneRows, row)) ? new[]{"+"} : new[]{""};
}
mdata.AddCategoryColumn("Histones", "", histoneCol);
// initialize the variables for the annotation rows
double[] totalProteinRow = new double[mdata.ExpressionColumnCount];
double[] totalMoleculesRow = new double[mdata.ExpressionColumnCount];
string[][] organismRow = new string[mdata.ExpressionColumnCount][];
double[] histoneMassRow = new double[mdata.ExpressionColumnCount];
double[] ploidyRow = new double[mdata.ExpressionColumnCount];
double[] cellVolumeRow = new double[mdata.ExpressionColumnCount];
double[] normalizationFactors = new double[columns.Count];
// calculate normalization factors for each column
for (int col = 0; col < columns.Count; col++){
string sampleName = sampleNames[col];
double[] column = columns[col];
// normalization factor to go from intensities to copies,
// needs to be determined either using the total protein or the histone scaling approach
double factor;
switch (param.GetSingleChoiceWithSubParams("Scaling mode").Value){
case 0: // total protein amount
double mwWeightedNormalizedSummedIntensities = 0;
for (int row = 0; row < mdata.RowCount; row++){
if (!double.IsNaN(column[row]) && !double.IsNaN(mw[row])){
mwWeightedNormalizedSummedIntensities += (column[row]/detectabilityNormFactor[row])*mw[row];
}
}
factor =
(param.GetSingleChoiceWithSubParams("Scaling mode")
.GetSubParameters()
.GetDoubleParam("Protein amount per cell [pg]")
.Value*1e-12*avogadro)/mwWeightedNormalizedSummedIntensities;
break;
case 1: // histone mode
double mwWeightedNormalizedSummedHistoneIntensities = 0;
foreach (int row in histoneRows){
if (!double.IsNaN(column[row]) && !double.IsNaN(mw[row])){
mwWeightedNormalizedSummedHistoneIntensities += (column[row]/detectabilityNormFactor[row])*mw[row];
}
}
double ploidy =
param.GetSingleChoiceWithSubParams("Scaling mode").GetSubParameters().GetDoubleParam("Ploidy").Value;
factor = (cValue*ploidy*avogadro)/mwWeightedNormalizedSummedHistoneIntensities;
break;
default:
factor = 1;
break;
}
normalizationFactors[col] = factor;
}
// check averaging mode
if (param.GetSingleChoiceWithSubParams("Averaging mode").Value == 1) // same factor for all
{
double factor = ArrayUtils.Mean(normalizationFactors);
for (int i = 0; i < normalizationFactors.Length; i++){
normalizationFactors[i] = factor;
}
}
if (param.GetSingleChoiceWithSubParams("Averaging mode").Value == 2) // same factor in each group
{
if (
param.GetSingleChoiceWithSubParams("Averaging mode").GetSubParameters().GetSingleChoiceParam("Grouping").Value ==
-1){
processInfo.ErrString = "No grouping selected.";
return;
}
string[][] groupNames =
mdata.GetCategoryRowAt(
param.GetSingleChoiceWithSubParams("Averaging mode").GetSubParameters().GetSingleChoiceParam("Grouping").Value);
string[] uniqueGroupNames = Unique(groupNames);
int[] grouping = new int[columns.Count];
for (int i = 0; i < columns.Count; i++){
if (intensityCols[i] >= mdata.ExpressionColumnCount){ // Numeric annotation columns cannot be grouped
grouping[i] = i;
continue;
}
if (ArrayUtils.Contains(uniqueGroupNames, groupNames[i][0])){
grouping[i] = ArrayUtils.IndexOf(uniqueGroupNames, groupNames[i][0]);
continue;
}
grouping[i] = i;
}
Dictionary<int, List<double>> factors = new Dictionary<int, List<double>>();
for (int i = 0; i < columns.Count; i++){
if (factors.ContainsKey(grouping[i])){
factors[grouping[i]].Add(normalizationFactors[i]);
} else{
factors.Add(grouping[i], new List<double>{normalizationFactors[i]});
}
}
double[] averagedNormalizationFactors = new double[columns.Count];
for (int i = 0; i < columns.Count; i++){
List<double> factor;
factors.TryGetValue(grouping[i], out factor);
averagedNormalizationFactors[i] = ArrayUtils.Mean(factor);
}
normalizationFactors = averagedNormalizationFactors;
}
// loop over all selected columns and calculate copy numbers
for (int col = 0; col < columns.Count; col++){
string sampleName = sampleNames[col];
double[] column = columns[col];
double factor = normalizationFactors[col];
double[] copyNumbers = new double[mdata.RowCount];
double[] concentrations = new double[mdata.RowCount]; // femtoliters
double[] massFraction = new double[mdata.RowCount];
double[] moleFraction = new double[mdata.RowCount];
double totalProtein = 0; // picograms
double histoneMass = 0; // picograms
double totalMolecules = 0;
for (int row = 0; row < mdata.RowCount; row++){
if (!double.IsNaN(column[row]) && !double.IsNaN(mw[row])){
copyNumbers[row] = (column[row]/detectabilityNormFactor[row])*factor;
totalMolecules += copyNumbers[row];
totalProtein += (copyNumbers[row]*mw[row]*1e12)/avogadro; // picograms
if (ArrayUtils.Contains(histoneRows, row)){
histoneMass += (copyNumbers[row]*mw[row]*1e12)/avogadro; // picograms
}
}
}
double totalVolume = (totalProtein/(param.GetDoubleParam("Total cellular protein concentration [g/l]").Value))*1000;
// femtoliters
for (int row = 0; row < mdata.RowCount; row++){
if (!double.IsNaN(column[row]) && !double.IsNaN(mw[row])){
concentrations[row] = ((copyNumbers[row]/(totalVolume*1e-15))/avogadro)*1e9; // nanomolar
massFraction[row] = (((copyNumbers[row]*mw[row]*1e12)/avogadro)/totalProtein)*1e6; // ppm
moleFraction[row] = (copyNumbers[row]/totalMolecules)*1e6; // ppm
}
}
string suffix = (sampleName == "") ? "" : " " + sampleName;
if (ArrayUtils.Contains(outputColumns, 0)){
mdata.AddNumericColumn("Copy number" + suffix, "", copyNumbers);
}
if (ArrayUtils.Contains(outputColumns, 1)){
mdata.AddNumericColumn("Concentration [nM]" + suffix, "", concentrations);
}
if (ArrayUtils.Contains(outputColumns, 2)){
mdata.AddNumericColumn("Abundance (mass/total mass) [*10^-6]" + suffix, "", massFraction);
}
if (ArrayUtils.Contains(outputColumns, 3)){
mdata.AddNumericColumn("Abundance (molecules/total molecules) [*10^-6]" + suffix, "", moleFraction);
}
double[] rank = ArrayUtils.Rank(copyNumbers);
double[] relativeRank = new double[mdata.RowCount];
double validRanks = mdata.RowCount;
for (int row = 0; row < mdata.RowCount; row++){
// remove rank for protein with no copy number information
if (double.IsNaN((copyNumbers[row])) || copyNumbers[row] == 0){
rank[row] = double.NaN;
validRanks--; // do not consider as valid
}
// invert ranking, so that rank 0 is the most abundant protein
rank[row] = mdata.RowCount - rank[row];
}
for (int row = 0; row < mdata.RowCount; row++){
relativeRank[row] = rank[row]/validRanks;
}
if (ArrayUtils.Contains(outputColumns, 4)){
mdata.AddNumericColumn("Copy number rank" + suffix, "", rank);
}
if (ArrayUtils.Contains(outputColumns, 5)){
mdata.AddNumericColumn("Relative copy number rank" + suffix, "", relativeRank);
}
if (intensityCols[col] < mdata.ExpressionColumnCount &&
param.GetSingleChoiceWithSubParams("Averaging mode").Value != 3){
totalProteinRow[intensityCols[col]] = Math.Round(totalProtein, 2);
totalMoleculesRow[intensityCols[col]] = Math.Round(totalMolecules, 0);
organismRow[intensityCols[col]] = new string[]{organism.name};
histoneMassRow[intensityCols[col]] = Math.Round(histoneMass, 4);
ploidyRow[intensityCols[col]] = Math.Round((histoneMass*1e-12)/cValue, 2);
cellVolumeRow[intensityCols[col]] = Math.Round(totalVolume, 2); // femtoliters
}
}
if (param.GetSingleChoiceWithSubParams("Averaging mode").Value != 3 && ArrayUtils.Contains(outputColumns, 6)){
mdata.AddNumericRow("Total protein [pg/cell]", "", totalProteinRow);
mdata.AddNumericRow("Total molecules per cell", "", totalMoleculesRow);
mdata.AddCategoryRow("Organism", "", organismRow);
mdata.AddNumericRow("Histone mass [pg/cell]", "", histoneMassRow);
mdata.AddNumericRow("Ploidy", "", ploidyRow);
mdata.AddNumericRow("Cell volume [fl]", "", cellVolumeRow);
}
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] cols = param1.GetMultiChoiceParam("Columns").Value;
int[] ops = param1.GetMultiChoiceParam("Operation").Value;
foreach (int t in ops){
double[][] vals = new double[cols.Length][];
for (int i = 0; i < cols.Length; i++){
double[][] x = mdata.MultiNumericColumns[cols[i]];
vals[i] = new double[x.Length];
for (int j = 0; j < vals[i].Length; j++){
vals[i][j] = operations[t](x[j]);
}
}
for (int i = 0; i < cols.Length; i++){
mdata.AddNumericColumn(mdata.MultiNumericColumnNames[cols[i]] + "_" + names[t], "", vals[i]);
}
}
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables, ProcessInfo processInfo)
{
int[] multiNumCols = param1.GetMultiChoiceParam("Multi-numeric columns").Value;
Array.Sort(multiNumCols);
int[] stringCols = param1.GetMultiChoiceParam("String columns").Value;
Array.Sort(stringCols);
HashSet<int> multinumCols2 = new HashSet<int>(multiNumCols);
HashSet<int> stringCols2 = new HashSet<int>(stringCols);
if (multiNumCols.Length + stringCols.Length == 0){
processInfo.ErrString = "Please select some columns.";
return;
}
int rowCount = GetNewRowCount(mdata, multiNumCols, stringCols);
float[,] expVals = new float[rowCount,mdata.ExpressionColumnCount];
List<string[]> stringC = new List<string[]>();
for (int i = 0; i < mdata.StringColumnCount; i++){
stringC.Add(new string[rowCount]);
}
List<double[]> numC = new List<double[]>();
for (int i = 0; i < mdata.NumericColumnCount; i++){
numC.Add(new double[rowCount]);
}
List<string[][]> catC = new List<string[][]>();
for (int i = 0; i < mdata.CategoryColumnCount; i++){
catC.Add(new string[rowCount][]);
}
List<double[][]> multiNumC = new List<double[][]>();
for (int i = 0; i < mdata.MultiNumericColumnCount; i++){
multiNumC.Add(new double[rowCount][]);
}
int count = 0;
for (int i = 0; i < mdata.RowCount; i++){
string err;
int entryCount = GetEntryCount(i, mdata, multiNumCols, stringCols, out err);
if (err != null){
processInfo.ErrString = err;
return;
}
bool empty = entryCount == 0;
entryCount = Math.Max(entryCount, 1);
for (int j = 0; j < entryCount; j++){
for (int k = 0; k < mdata.ExpressionColumnCount; k++){
expVals[count + j, k] = mdata[i, k];
}
for (int k = 0; k < mdata.NumericColumnCount; k++){
numC[k][count + j] = mdata.NumericColumns[k][i];
}
for (int k = 0; k < mdata.CategoryColumnCount; k++){
catC[k][count + j] = mdata.CategoryColumns[k][i];
}
}
for (int k = 0; k < mdata.MultiNumericColumnCount; k++){
if (multinumCols2.Contains(k)){
if (empty){
multiNumC[k][count] = new double[0];
} else{
double[] vals = mdata.MultiNumericColumns[k][i];
for (int j = 0; j < entryCount; j++){
multiNumC[k][count + j] = new[]{vals[j]};
}
}
} else{
for (int j = 0; j < entryCount; j++){
multiNumC[k][count + j] = mdata.MultiNumericColumns[k][i];
}
}
}
for (int k = 0; k < mdata.StringColumnCount; k++){
if (stringCols2.Contains(k)){
if (empty){
stringC[k][count] = "";
} else{
string[] vals = mdata.StringColumns[k][i].Split(';');
for (int j = 0; j < entryCount; j++){
stringC[k][count + j] = vals[j];
}
}
} else{
for (int j = 0; j < entryCount; j++){
stringC[k][count + j] = mdata.StringColumns[k][i];
}
}
}
count += entryCount;
}
int[] multiNumComplement = ArrayUtils.Complement(multiNumCols, mdata.MultiNumericColumnCount);
List<double[][]> toBeTransformed = ArrayUtils.SubList(multiNumC, multiNumCols);
multiNumC = ArrayUtils.SubList(multiNumC, multiNumComplement);
foreach (double[][] d in toBeTransformed){
numC.Add(Transform(d));
}
mdata.SetData(mdata.Name, mdata.ExpressionColumnNames, expVals, mdata.StringColumnNames, stringC,
mdata.CategoryColumnNames, catC,
new List<string>(ArrayUtils.Concat(mdata.NumericColumnNames,
ArrayUtils.SubList(mdata.MultiNumericColumnNames, multiNumCols))), numC,
new List<string>(ArrayUtils.SubArray(mdata.MultiNumericColumnNames, multiNumComplement)), multiNumC);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
string regexStr = param.GetStringParam("Regular expression").Value;
Regex regex = new Regex(regexStr);
int[] inds = param.GetMultiChoiceParam("Columns").Value;
foreach (int ind in inds){
ProcessCol(mdata.StringColumns[ind], regex);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int minCount = param.GetIntParam("Min. count").Value;
int selCol = param.GetSingleChoiceParam("Selection").Value;
string value = param.GetStringParam("Value").Value;
int[] catIndices = param.GetMultiChoiceParam("Categories").Value;
bool[] selection = null;
if (selCol < mdata.CategoryColumnCount){
selection = new bool[mdata.RowCount];
string[][] x = mdata.GetCategoryColumnAt(selCol);
for (int i = 0; i < selection.Length; i++){
if (x[i] != null){
for (int j = 0; j < x[i].Length; j++){
if (x[i][j].Equals(value)){
selection[i] = true;
break;
}
}
}
}
}
CountingResult result = CountCategories(mdata, selection, selCol, catIndices);
CreateMatrixData(result, mdata, minCount, selection);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
string colName = param.GetStringParam("Name of new column").Value;
int[] columns = param.GetMultiChoiceParam("Categories").Value;
bool inverse = param.GetBoolParam("Inverse").Value;
int[] catCols;
int[] stringCols;
Split(columns, out catCols, out stringCols, mdata.CategoryColumnCount);
string[] word1 = param.GetMultiStringParam("Search terms").Value;
if (word1.Length == 0){
processInfo.ErrString = "Please specify one or more search terms.";
return;
}
if (string.IsNullOrEmpty(colName)){
colName = word1[0];
}
string[] word = new string[word1.Length];
for (int i = 0; i < word.Length; i++){
word[i] = word1[i].ToLower().Trim();
}
bool[] indicator = new bool[mdata.RowCount];
foreach (int col in catCols){
string[][] cat = mdata.GetCategoryColumnAt(col);
for (int i = 0; i < cat.Length; i++){
foreach (string s in cat[i]){
foreach (string s1 in word){
if (s.ToLower().Contains(s1)){
indicator[i] = true;
break;
}
}
}
}
}
foreach (string[] txt in stringCols.Select(col => mdata.StringColumns[col])){
for (int i = 0; i < txt.Length; i++){
string s = txt[i];
foreach (string s1 in word){
if (s.ToLower().Contains(s1)){
indicator[i] = true;
break;
}
}
}
}
string[][] newCol = new string[indicator.Length][];
for (int i = 0; i < newCol.Length; i++){
bool yes = inverse ? !indicator[i] : indicator[i];
newCol[i] = yes ? new[]{"+"} : new string[0];
}
mdata.AddCategoryColumn(colName, "", newCol);
}
