public MatrixDataTable(IMatrixData mdata)
: base(mdata.Name, mdata.Description, mdata.RowCount)
{
this.mdata = mdata;
CreateColumns();
CreateAnnotationRows();
}
public Parameters GetParameters(IMatrixData[] inputData, ref string errString)
{
IMatrixData matrixData1 = inputData[0];
IMatrixData matrixData2 = inputData[1];
return
new Parameters(new Parameter[]{
new SingleChoiceParam("Column in matrix 1 to be edited"){
Values = matrixData1.StringColumnNames,
Value = 0,
Help =
"The column in the first matrix in which strings will be replaced " +
"according to the key-value table specified in matrix 2."
},
new SingleChoiceParam("Keys in matrix 2"){
Values = matrixData2.StringColumnNames,
Value = 0,
Help = "The keys for the replacement table."
},
new SingleChoiceParam("Values in matrix 2"){
Values = matrixData2.StringColumnNames,
Value = 1,
Help = "The values for the replacement table."
}
});
}
public static void DivideByColumn(IMatrixData data, int index)
{
int p = data.RowCount;
int n = data.ExpressionColumnCount;
float[,] newEx = new float[p,n - 1];
for (int i = 0; i < p; i++){
for (int j = 0; j < index; j++){
newEx[i, j] = data[i, j] - data[i, index];
}
for (int j = index + 1; j < n; j++){
newEx[i, j - 1] = data[i, j] - data[i, index];
}
}
bool[,] newImp = new bool[p,n - 1];
for (int i = 0; i < p; i++){
for (int j = 0; j < index; j++){
newImp[i, j] = data.IsImputed[i, j] || data.IsImputed[i, index];
}
for (int j = index + 1; j < n; j++){
newImp[i, j - 1] = data.IsImputed[i, j] || data.IsImputed[i, index];
}
}
data.ExpressionValues = newEx;
data.IsImputed = newImp;
data.ExpressionColumnNames.RemoveAt(index);
data.ExpressionColumnDescriptions.RemoveAt(index);
for (int i = 0; i < data.CategoryRowCount; i++){
data.CategoryColumns[i] = ArrayUtils.RemoveAtIndex(data.CategoryColumns[i], index);
}
for (int i = 0; i < data.NumericRowCount; i++){
data.NumericColumns[i] = ArrayUtils.RemoveAtIndex(data.NumericColumns[i], index);
}
}
public static void SubtractValues(bool rows, Func<double[], double> summarize, IMatrixData data)
{
if (rows){
for (int i = 0; i < data.RowCount; i++){
List<double> vals = new List<double>();
for (int j = 0; j < data.ExpressionColumnCount; j++){
double q = data[i, j];
if (!double.IsNaN(q) && !double.IsInfinity(q)){
vals.Add(q);
}
}
double med = summarize(vals.ToArray());
for (int j = 0; j < data.ExpressionColumnCount; j++){
data[i, j] -= (float) med;
}
}
} else{
for (int j = 0; j < data.ExpressionColumnCount; j++){
List<double> vals = new List<double>();
for (int i = 0; i < data.RowCount; i++){
double q = data[i, j];
if (!double.IsNaN(q) && !double.IsInfinity(q)){
vals.Add(q);
}
}
double med = summarize(vals.ToArray());
for (int i = 0; i < data.RowCount; i++){
data[i, j] -= (float) med;
}
}
}
}
public static string ReplaceMissingsByGaussianWholeMatrix(double width, double shift, IMatrixData data, int[] colInds)
{
List<float> allValues = new List<float>();
for (int i = 0; i < data.RowCount; i++){
foreach (int t in colInds){
float x = GetValue(data, i, t);
if (!float.IsNaN(x) && !float.IsInfinity(x)){
allValues.Add(x);
}
}
}
double stddev;
double mean = ArrayUtils.MeanAndStddev(allValues.ToArray(), out stddev);
if (double.IsNaN(mean) || double.IsInfinity(mean) || double.IsNaN(stddev) || double.IsInfinity(stddev)){
return "Imputation failed since mean and standard deviation could not be calculated.";
}
double m = mean - shift*stddev;
double s = stddev*width;
Random2 r = new Random2();
for (int i = 0; i < data.RowCount; i++){
foreach (int colInd in colInds){
float x = GetValue(data, i, colInd);
if (float.IsNaN(x) || float.IsInfinity(x)){
if (colInd < data.ColumnCount){
data.Values.Set(i, colInd, (float) r.NextGaussian(m, s));
data.IsImputed[i, colInd] = true;
} else{
data.NumericColumns[colInd - data.ColumnCount][i] = r.NextGaussian(m, s);
}
}
}
}
return null;
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
List<string> colChoice = mdata.StringColumnNames;
int colInd = 0;
for (int i = 0; i < colChoice.Count; i++){
if (colChoice[i].ToUpper().Equals("UNIPROT")){
colInd = i;
break;
}
}
int colSeqInd = 0;
for (int i = 0; i < colChoice.Count; i++){
if (colChoice[i].ToUpper().Equals("SEQUENCE WINDOW")){
colSeqInd = i;
break;
}
}
return
new Parameters(
new SingleChoiceParam("Uniprot column"){
Value = colInd,
Help = "Specify here the column that contains Uniprot identifiers.",
Values = colChoice
},
new SingleChoiceParam("Sequence window"){
Value = colSeqInd,
Help = "Specify here the column that contains the sequence windows around the site.",
Values = colChoice
});
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
string[] vals = ArrayUtils.Concat(mdata.ColumnNames, mdata.NumericColumnNames);
int[] sel1 = vals.Length > 0 ? new[]{0} : new int[0];
int[] sel2 = vals.Length > 1 ? new[]{1} : (vals.Length > 0 ? new[]{0} : new int[0]);
return
new Parameters(new Parameter[]{
new MultiChoiceParam("x", sel1){
Values = vals,
Repeats = true,
Help =
"Colums for the first dimension. Multiple choices can be made leading to the creation of multiple density maps."
},
new MultiChoiceParam("y", sel2){
Values = vals,
Repeats = true,
Help = "Colums for the second dimension. The number has to be the same as for the 'Column 1' parameter."
},
new IntParam("Number of points", 300){
Help =
"This parameter defines the resolution of the density map. It specifies the number of pixels per dimension. Large " +
"values may lead to increased computing times."
},
new SingleChoiceParam("Distribution type"){Values = new[]{"P(x,y)", "P(y|x)", "P(x|y)", "P(x,y)/(P(x)*P(y))"}}
});
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
Random2 rand = new Random2();
double std = param.GetParam<double>("Standard deviation").Value;
int[] inds = param.GetParam<int[]>("Columns").Value;
List<int> mainInds = new List<int>();
List<int> numInds = new List<int>();
foreach (int ind in inds){
if (ind < mdata.ColumnCount){
mainInds.Add(ind);
} else{
numInds.Add(ind - mdata.ColumnCount);
}
}
foreach (int j in mainInds){
for (int i = 0; i < mdata.RowCount; i++){
mdata.Values.Set(i, j, mdata.Values.Get(i, j) + (float) rand.NextGaussian(0, std));
}
}
foreach (int j in numInds){
for (int i = 0; i < mdata.RowCount; i++){
mdata.NumericColumns[j][i] += (float) rand.NextGaussian(0, std);
}
}
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
List<Parameter> par = new List<Parameter>();
foreach (string t in mdata.ColumnNames){
string help = "Specify the new name for the column '" + t + "'.";
par.Add(new StringParam(t){Value = t, Help = help});
}
foreach (string t in mdata.NumericColumnNames){
string help = "Specify the new name for the column '" + t + "'.";
par.Add(new StringParam(t){Value = t, Help = help});
}
foreach (string t in mdata.CategoryColumnNames){
string help = "Specify the new name for the column '" + t + "'.";
par.Add(new StringParam(t){Value = t, Help = help});
}
foreach (string t in mdata.StringColumnNames){
string help = "Specify the new name for the column '" + t + "'.";
par.Add(new StringParam(t){Value = t, Help = help});
}
foreach (string t in mdata.MultiNumericColumnNames){
string help = "Specify the new name for the column '" + t + "'.";
par.Add(new StringParam(t){Value = t, Help = help});
}
return new Parameters(par);
}
public void UnitVectors(bool rows, IMatrixData data)
{
if (rows){
for (int i = 0; i < data.RowCount; i++){
double len = 0;
for (int j = 0; j < data.ColumnCount; j++){
double q = data.Values.Get(i, j);
len += q*q;
}
len = Math.Sqrt(len);
for (int j = 0; j < data.ColumnCount; j++){
data.Values.Set(i, j, data.Values.Get(i, j)/(float) len);
}
}
} else{
for (int j = 0; j < data.ColumnCount; j++){
double len = 0;
for (int i = 0; i < data.RowCount; i++){
double q = data.Values.Get(i, j);
len += q*q;
}
len = Math.Sqrt(len);
for (int i = 0; i < data.RowCount; i++){
data.Values.Set(i, j, data.Values.Get(i, j) / (float) len);
}
}
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
const bool rows = false;
bool percentage;
int minValids = PerseusPluginUtils.GetMinValids(param, out percentage);
ParameterWithSubParams<int> modeParam = param.GetParamWithSubParams<int>("Mode");
int modeInd = modeParam.Value;
if (modeInd != 0 && mdata.CategoryRowNames.Count == 0){
processInfo.ErrString = "No grouping is defined.";
return;
}
if (modeInd != 0){
processInfo.ErrString = "Group-wise filtering can only be appled to rows.";
return;
}
FilteringMode filterMode;
double threshold;
double threshold2;
PerseusPluginUtils.ReadValuesShouldBeParams(param, out filterMode, out threshold, out threshold2);
if (modeInd != 0){
//TODO
} else{
PerseusPluginUtils.NonzeroFilter1(rows, minValids, percentage, mdata, param, threshold, threshold2, filterMode);
}
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
List<string> numRows = mdata.NumericRowNames;
List<string> multiNumRows = mdata.MultiNumericRowNames;
List<string> catRows = mdata.CategoryRowNames;
List<string> textRows = mdata.StringRowNames;
return
new Parameters(new Parameter[]{
new MultiChoiceParam("Numerical rows"){
Value = ArrayUtils.ConsecutiveInts(numRows.Count),
Values = numRows,
Help = "Specify here the new order in which the numerical rows should appear."
},
new MultiChoiceParam("Multi-numerical rows"){
Value = ArrayUtils.ConsecutiveInts(multiNumRows.Count),
Values = multiNumRows,
Help = "Specify here the new order in which the numerical rows should appear."
},
new MultiChoiceParam("Categorical rows"){
Value = ArrayUtils.ConsecutiveInts(catRows.Count),
Values = catRows,
Help = "Specify here the new order in which the categorical rows should appear."
},
new MultiChoiceParam("Text rows"){
Value = ArrayUtils.ConsecutiveInts(textRows.Count),
Values = textRows,
Help = "Specify here the new order in which the text rows should appear."
}
});
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
Parameters[] subParams = new Parameters[mdata.CategoryColumnCount];
for (int i = 0; i < mdata.CategoryColumnCount; i++){
string[] values = mdata.GetCategoryColumnValuesAt(i);
int[] sel = values.Length == 1 ? new[]{0} : new int[0];
subParams[i] =
new Parameters(new Parameter[]{
new MultiChoiceParam("Values", sel){
Values = values,
Help = "The value that should be present to discard/keep the corresponding row."
}
});
}
return
new Parameters(new SingleChoiceWithSubParams("Column"){
Values = mdata.CategoryColumnNames,
SubParams = subParams,
Help = "The categorical column that the filtering should be based on.",
ParamNameWidth = 50,
TotalWidth = 731
}, new SingleChoiceParam("Mode"){
Values = new[]{"Remove matching rows", "Keep matching rows"},
Help =
"If 'Remove matching rows' is selected, rows having the values specified above will be removed while " +
"all other rows will be kept. If 'Keep matching rows' is selected, the opposite will happen."
}, PerseusPluginUtils.GetFilterModeParam(true));
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
Parameter<int> access = param.GetParam<int>("Matrix access");
bool rows = access.Value == 0;
UnitVectors(rows, mdata);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
bool rows = param.GetParam<int>("Matrix access").Value == 0;
double min = param.GetParam<double>("Minimum").Value;
double max = param.GetParam<double>("Maximum").Value;
MapToInterval1(rows, mdata, min, max, processInfo.NumThreads);
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
return
new Parameters(new Parameter[]{
new StringParam("Find what"), new SingleChoiceParam("Look in"){Values = mdata.StringColumnNames},
new BoolParam("Match case"){Value = true}, new BoolParam("Match whole word")
});
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
return
new Parameters(new DoubleParam("Standard deviation", 0.1){Help = "Standard deviation of the noise distribution."},
new MultiChoiceParam("Columns", ArrayUtils.ConsecutiveInts(mdata.ColumnCount)){
Values = ArrayUtils.Concat(mdata.ColumnNames, mdata.NumericColumnNames)
});
}
public SelectRowsManuallyControl(IMatrixData mdata, Action<IData> createNewMatrix)
{
InitializeComponent();
this.mdata = mdata;
this.createNewMatrix = createNewMatrix;
matrixDataGridView.HasRemoveRowsMenuItems = false;
matrixDataGridView.TableModel = new MatrixDataTable(mdata);
}
public static void MapToInterval1(bool rows, IMatrixData data, double min, double max, int nthreads)
{
if (rows){
new ThreadDistributor(nthreads, data.RowCount, i => Calc1(i, data, min, max)).Start();
} else{
new ThreadDistributor(nthreads, data.ColumnCount, j => Calc2(j, data, min, max)).Start();
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
string[] mods = param.GetParam<int[]>("Modifications").StringValue.Split(new[]{';'},
StringSplitOptions.RemoveEmptyEntries);
string[] up = mdata.StringColumns[param.GetParam<int>("Uniprot column").Value];
string[][] uprot = new string[up.Length][];
for (int i = 0; i < up.Length; i++){
uprot[i] = up[i].Length > 0 ? up[i].Split(';') : new string[0];
}
double[][] c = new double[mods.Length][];
for (int index = 0; index < mods.Length; index++){
string mod = mods[index];
string filename = PhosphoSitePlusParser.GetFilenameForMod(mod);
if (filename == null){
processInfo.ErrString = "File does not exist.";
return;
}
string[] seqWins;
string[] accs;
string[] pubmedLtp;
string[] pubmedMs2;
string[] cstMs2;
string[] species;
PhosphoSitePlusParser.ParseKnownMods(filename, out seqWins, out accs, out pubmedLtp, out pubmedMs2, out cstMs2, out species);
for (int i = 0; i < seqWins.Length; i++){
seqWins[i] = seqWins[i].ToUpper();
}
Dictionary<string, HashSet<string>> counts = new Dictionary<string, HashSet<string>>();
for (int i = 0; i < accs.Length; i++){
string acc = accs[i];
if (!counts.ContainsKey(acc)){
counts.Add(acc, new HashSet<string>());
}
counts[acc].Add(seqWins[i]);
}
c[index] = new double[up.Length];
for (int i = 0; i < up.Length; i++){
c[index][i] = CountSites(uprot[i], counts);
}
}
string[][] catCol = new string[up.Length][];
for (int i = 0; i < catCol.Length; i++){
List<string> x = new List<string>();
for (int j = 0; j < mods.Length; j++){
if (c[j][i] > 0){
x.Add(mods[j]);
}
}
x.Sort();
catCol[i] = x.ToArray();
}
mdata.AddCategoryColumn("Known modifications", "Known modifications", catCol);
for (int i = 0; i < mods.Length; i++){
mdata.AddNumericColumn(mods[i] + " count", mods[i] + " count", c[i]);
}
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
return
new Parameters(new Parameter[]{
new MultiChoiceParam("Columns"){Values = mdata.StringColumnNames},
new StringParam("Regular expression", "^([^;]+)"), new BoolParam("Keep original columns", false),
new BoolParam("Strings separated by semicolons are independent", false)
});
}
public Parameters GetCreateParameters(IMatrixData mdata)
{
List<Parameter> par = new List<Parameter>
{new StringParam("Row name"){Value = "Group1", Help = "Name of the new category annotation row."}};
foreach (string t in mdata.ExpressionColumnNames){
string help = "Specify a value for the column '" + t + "'.";
par.Add(new StringParam(t){Value = t, Help = help});
}
return new Parameters(par);
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
return
new Parameters(new Parameter[]{
new SingleChoiceParam("New column names"){
Values = mdata.StringColumnNames,
Help = "Select the column that should become the column names of the transposed matrix."
}
});
}
public Parameters GetDeleteParameters(IMatrixData mdata)
{
List<Parameter> par = new List<Parameter>{
new SingleChoiceParam("Numerical row"){
Values = mdata.NumericRowNames,
Help = "Select the numerical row that should be deleted."
}
};
return new Parameters(par);
}
public Parameters GetEditParameters(IMatrixData mdata, int ind)
{
List<Parameter> par = new List<Parameter>();
for (int i = 0; i < mdata.ColumnCount; i++){
string t = mdata.ColumnNames[i];
string help = "Specify a numerical value for the column '" + t + "'.";
par.Add(new DoubleParam(t, mdata.NumericRows[ind][i]){Help = help});
}
return new Parameters(par);
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
return
new Parameters(new Parameter[]{
new SingleChoiceParam("Indicator column"){Values = mdata.CategoryColumnNames},
new StringParam("Value", "+"){
Help = "Rows matching this term in the indicator column will be used as control for the normalization."
}
});
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
return
new Parameters(new Parameter[]{
new SingleChoiceParam("Matrix access"){
Values = new[]{"Rows", "Columns"},
Help = "Specifies if the analysis is performed on the rows or the columns of the matrix."
}
});
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
var vals = param.GetParam<Tuple<Regex, string>>("Regex").Value;
var pattern = vals.Item1;
string replacementStr = vals.Item2;
for (int i = 0; i < mdata.ColumnCount; i++){
mdata.ColumnNames[i] = pattern.Replace(mdata.ColumnNames[i], replacementStr);
}
}
public static void ReplaceMissingsByVal(float value, IMatrixData data)
{
for (int i = 0; i < data.RowCount; i++){
for (int j = 0; j < data.ExpressionColumnCount; j++){
if (float.IsNaN(data[i, j])){
data[i, j] = value;
data.IsImputed[i, j] = true;
}
}
}
}
public MatrixMultiStream(IMatrixData[] inputData, bool[] hasHeader)
{
_inputData = inputData;
_row = new int[inputData.Length];
for (int i = 0; i < inputData.Length; i++){
_row[i] = hasHeader[i] ? -1 : 0;
}
_total = GetLength(inputData, _row);
}
public Parameters GetCreateParameters(IMatrixData mdata)
{
List <Parameter> par = new List <Parameter> {
new StringParam("Row name")
{
Value = "Group1", Help = "Name of the new category annotation row."
}
};
foreach (string t in mdata.ColumnNames)
{
string help = "Specify a value for the column '" + t + "'.";
par.Add(new StringParam(t)
{
Value = t, Help = help
});
}
return(new Parameters(par));
}
public static int[] GetIndicesOfCol(IMatrixData data, string categoryName, HashSet <string> values)
{
int index = GetIndexOfCol(data, categoryName);
List <int> result = new List <int>();
for (int i = 0; i < data.ColumnCount; i++)
{
string[] s = data.GetCategoryRowEntryAt(index, i);
foreach (string s1 in s)
{
if (values.Contains(s1))
{
result.Add(i);
break;
}
}
}
return(result.ToArray());
}
public Parameters GetEditParameters(IMatrixData mdata)
{
Parameters[] subParams = new Parameters[mdata.CategoryRowCount];
for (int i = 0; i < subParams.Length; i++)
{
subParams[i] = GetEditParameters(mdata, i);
}
List <Parameter> par = new List <Parameter> {
new SingleChoiceWithSubParams("Category row")
{
Values = mdata.CategoryRowNames,
SubParams = subParams,
Help = "Select the category row that should be edited.",
ParamNameWidth = 50
}
};
return(new Parameters(par));
}
public static int[] GetIndicesOf(IMatrixData data, string categoryName, string value)
{
int index = GetIndexOf(data, categoryName);
List <int> result = new List <int>();
for (int i = 0; i < data.RowCount; i++)
{
string[] s = data.GetCategoryColumnEntryAt(index, i);
foreach (string s1 in s)
{
if (s1.Equals(value))
{
result.Add(i);
break;
}
}
}
return(result.ToArray());
}
public static Parameter[] CreateCopyParameters(IMatrixData matrixData2)
{
return(new Parameter[] {
new MultiChoiceParam("Copy main columns")
{
Values = matrixData2.ColumnNames,
Value = new int[0],
Help = "Main columns of the second matrix that should be added to the first matrix."
},
new SingleChoiceParam("Combine copied main values")
{
Values = new[] { "Median", "Mean", "Minimum", "Maximum", "Sum", "Keep separate" },
Help =
"In case multiple rows of the second matrix match to a row of the first matrix, how should multiple " +
"values be combined?"
},
new MultiChoiceParam("Copy categorical columns")
{
Values = matrixData2.CategoryColumnNames,
Value = new int[0],
Help = "Categorical columns of the second matrix that should be added to the first matrix."
},
new MultiChoiceParam("Copy text columns")
{
Values = matrixData2.StringColumnNames,
Value = new int[0],
Help = "Text columns of the second matrix that should be added to the first matrix."
},
new MultiChoiceParam("Copy numerical columns")
{
Values = matrixData2.NumericColumnNames,
Value = new int[0],
Help = "Numerical columns of the second matrix that should be added to the first matrix."
},
new SingleChoiceParam("Combine copied numerical values")
{
Values = new[] { "Median", "Mean", "Minimum", "Maximum", "Sum", "Keep separate" },
Help =
"In case multiple rows of the second matrix match to a row of the first matrix, how should multiple " +
"numerical values be combined?"
}
});
}
private static void Calc2(int j, Func <double[], double> summarize, IMatrixData data)
{
List <double> vals = new List <double>();
for (int i = 0; i < data.RowCount; i++)
{
double q = data.Values.Get(i, j);
if (!double.IsNaN(q) && !double.IsInfinity(q))
{
vals.Add(q);
}
}
double med = summarize(vals.ToArray());
for (int i = 0; i < data.RowCount; i++)
{
data.Values.Set(i, j, data.Values.Get(i, j) / med);
}
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
SingleChoiceWithSubParams scwsp = new SingleChoiceWithSubParams("Action")
{
Values =
new[] {
"Create", "Create from experiment name", "Edit", "Rename", "Delete", "Write template file", "Read from file"
},
SubParams = new[] {
GetCreateParameters(mdata), GetCreateFromExperimentNamesParameters(mdata, ref errorString),
GetEditParameters(mdata), GetRenameParameters(mdata), GetDeleteParameters(mdata),
GetWriteTemplateFileParameters(mdata), GetReadFromFileParameters(mdata)
},
ParamNameWidth = 136,
TotalWidth = 731
};
return(new Parameters(new Parameter[] { scwsp }));
}
public IMatrixData ProcessData(IMatrixData[] inputData, Parameters parameters, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
IMatrixData mdata1 = inputData[0];
IMatrixData mdata2 = inputData[1];
var matching = ParseMatchingColumns(parameters);
var(indexMap, unmappedRightIndices) = GetIndexMap(mdata1, mdata2, matching.first, matching.second, matching.ignoreCase);
var result = (IMatrixData)mdata1.Clone();
result.Origin = "Combination";
if (matching.outer)
{
var extendedIndexMap = new int[indexMap.Length + unmappedRightIndices.Length][];
Array.Copy(indexMap, extendedIndexMap, indexMap.Length);
for (int i = 0; i < unmappedRightIndices.Length; i++)
{
var idx = indexMap.Length + i;
extendedIndexMap[idx] = new[] { unmappedRightIndices[i] };
}
result.AddEmptyRows(unmappedRightIndices.Length);
UpdateIdColumns(result, mdata2, indexMap.Length, unmappedRightIndices, matching.first, matching.second);
indexMap = extendedIndexMap;
}
var addIndicator = parameters.GetParam <bool>("Add indicator").Value;
if (addIndicator)
{
AddIndicator(result, mdata2, indexMap);
}
var addRowIndex = parameters.GetParam <bool>("Add original row numbers").Value;
if (addRowIndex)
{
result.AddMultiNumericColumn("Original row numbers", "", indexMap.Select(rows => rows.Select(Convert.ToDouble).ToArray()).ToArray());
}
var(main, text, numeric, category) = ParseCopyParameters(parameters);
SetAnnotationRows(result, mdata1, mdata2, main.copy);
AddMainColumns(result, mdata2, indexMap, main.copy, GetAveraging(main.combine));
AddAnnotationColumns(result, mdata2, indexMap, text, numeric, category);
return(result);
}
public IMatrixData Add(IMatrixData current, IMatrixData value)
{
if (value is FloatNumberMatrixData floatNumberData)
{
return(this.AddFloat((UnsafeRGBMatrixData)current, floatNumberData));
}
if (value is UnsafeRGBMatrixData rgbData)
{
return(this.AddRgb((UnsafeRGBMatrixData)current, rgbData));
}
if (value is IntegerNumberMatrixData integerNumberMatrixData)
{
return(this.AddInteger((UnsafeRGBMatrixData)current, integerNumberMatrixData));
}
throw new NotImplementedException();
}
private static Dictionary <string, string> GetMap(IMatrixData mdata2, Parameters parameters)
{
string[] keys = mdata2.StringColumns[parameters.GetParam <int>("Keys in matrix 2").Value];
string[] values = mdata2.StringColumns[parameters.GetParam <int>("Values in matrix 2").Value];
Dictionary <string, string> map = new Dictionary <string, string>();
for (int i = 0; i < keys.Length; i++)
{
if (string.IsNullOrWhiteSpace(keys[i]))
{
continue;
}
if (!map.ContainsKey(keys[i]))
{
map.Add(keys[i], values[i]);
}
}
return(map);
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
List <string> exCols = mdata.ColumnNames;
List <string> numCols = mdata.NumericColumnNames;
List <string> multiNumCols = mdata.MultiNumericColumnNames;
List <string> catCols = mdata.CategoryColumnNames;
List <string> textCols = mdata.StringColumnNames;
return
(new Parameters(new Parameter[] {
new MultiChoiceParam("Main columns")
{
Value = new int[0],
Values = exCols,
Help = "Specify here the main columns that should be duplicated."
},
new MultiChoiceParam("Numerical columns")
{
Value = new int[0],
Values = numCols,
Help = "Specify here the numerical columns that should be duplicated."
},
new MultiChoiceParam("Multi-numerical columns")
{
Value = new int[0],
Values = multiNumCols,
Help = "Specify here the multi-numerical columns that should be duplicated."
},
new MultiChoiceParam("Categorical columns")
{
Value = new int[0],
Values = catCols,
Help = "Specify here the categorical columns that should be duplicated."
},
new MultiChoiceParam("Text columns")
{
Value = new int[0],
Values = textCols,
Help = "Specify here the text columns that should be duplicated."
}
}));
}
public static void CombineRows(this IMatrixData mdata, List <int> rowIdxs, Func <double[], double> combineNumeric,
Func <string[], string> combineString, Func <string[][], string[]> combineCategory,
Func <double[][], double[]> combineMultiNumeric)
{
if (!rowIdxs.Any())
{
return;
}
int resultRow = rowIdxs[0];
for (int i = 0; i < mdata.Values.ColumnCount; i++)
{
BaseVector column = mdata.Values.GetColumn(i);
BaseVector values = column.SubArray(rowIdxs);
mdata.Values[resultRow, i] = combineNumeric(ArrayUtils.ToDoubles(values));
}
for (int i = 0; i < mdata.NumericColumnCount; i++)
{
double[] column = mdata.NumericColumns[i];
double[] values = ArrayUtils.SubArray(column, rowIdxs);
column[resultRow] = combineNumeric(values);
}
for (int i = 0; i < mdata.StringColumnCount; i++)
{
string[] column = mdata.StringColumns[i];
string[] values = ArrayUtils.SubArray(column, rowIdxs);
column[resultRow] = combineString(values);
}
for (int i = 0; i < mdata.CategoryColumnCount; i++)
{
string[][] column = mdata.GetCategoryColumnAt(i);
string[][] values = ArrayUtils.SubArray(column, rowIdxs);
column[resultRow] = combineCategory(values);
mdata.SetCategoryColumnAt(column, i);
}
for (int i = 0; i < mdata.MultiNumericColumnCount; i++)
{
double[][] column = mdata.MultiNumericColumns[i];
double[][] values = ArrayUtils.SubArray(column, rowIdxs);
column[resultRow] = combineMultiNumeric(values);
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
double[] dm = new double[mdata.ColumnCount];
double[] dp = new double[mdata.ColumnCount];
for (int i = 0; i < mdata.ColumnCount; i++)
{
List <float> v = new List <float>();
foreach (double f in mdata.Values.GetColumn(i))
{
if (!double.IsNaN(f) && !double.IsInfinity(f))
{
v.Add((float)f);
}
}
float[] d = v.ToArray();
float[] q = ArrayUtils.Quantiles(d, new[] { 0.25, 0.5, 0.75 });
for (int j = 0; j < mdata.RowCount; j++)
{
mdata.Values[j, i] -= q[1];
}
dm[i] = q[1] - q[0];
dp[i] = q[2] - q[1];
}
double adm = ArrayUtils.Median(dm);
double adp = ArrayUtils.Median(dp);
for (int i = 0; i < mdata.ColumnCount; i++)
{
for (int j = 0; j < mdata.RowCount; j++)
{
if (mdata.Values[j, i] < 0)
{
mdata.Values[j, i] = (float)(mdata.Values[j, i] * adm / dm[i]);
}
else
{
mdata.Values[j, i] = (float)(mdata.Values[j, i] * adp / dp[i]);
}
}
}
}
//this function is modified from PerseusPluginLib/Load/UnstructuredTxtUpload.cs LoadSplit function
//obtains the output from fdr.exe (so only applicable to PECA CORE and N)
public static void GetOutput(IMatrixData mdata, Parameters param, Parameters dataParam, string filename, string geneName, string expSeries1 = "Expression Series 1", int numOfSeries = 2)
{
char separator = '\t';
//gene name column name is not included in the file so need to replace it
//gene name
ReplaceFirstLine(filename, geneName);
string[] colNames = TabSep.GetColumnNames(filename, 0, PerseusUtils.commentPrefix,
PerseusUtils.commentPrefixExceptions, null, separator);
string[][] cols = TabSep.GetColumns(colNames, filename, 0, PerseusUtils.commentPrefix,
PerseusUtils.commentPrefixExceptions, separator);
int nrows = TabSep.GetRowCount(filename);
string[] expressionColumnsNames = ArrayUtils.Concat(mdata.ColumnNames, mdata.NumericColumnNames);
mdata.Clear();
mdata.Name = "PECA Analysis";
mdata.Values.Init(nrows, 0);
mdata.SetAnnotationColumns(new List <string>(colNames), new List <string>(colNames), new List <string[]>(cols), new List <string>(),
new List <string>(), new List <string[][]>(), new List <string>(), new List <string>(), new List <double[]>(),
new List <string>(), new List <string>(), new List <double[][]>());
//be careful with changes of Number of time points in the future
int numOfExpCols = numOfSeries * dataParam.GetParam <int[]>(expSeries1).Value.Length;
//file format is structured so that expressions columns are before numeric ones
//so convert the numeric ones before expression columns
//first column guaranteed to be the name column
int[] expList = Enumerable.Range(1, numOfExpCols).ToArray();
int[] numericList = Enumerable.Range(numOfExpCols + 1, colNames.Count() - numOfExpCols - 1).ToArray();
StringToNumerical(numericList, mdata);
StringToExpression(expList, mdata);
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
List <string> exCols = mdata.ColumnNames;
List <string> numCols = mdata.NumericColumnNames;
List <string> multiNumCols = mdata.MultiNumericColumnNames;
List <string> catCols = mdata.CategoryColumnNames;
List <string> textCols = mdata.StringColumnNames;
return
(new Parameters(new Parameter[] {
new MultiChoiceParam("Main columns")
{
Value = ArrayUtils.ConsecutiveInts(exCols.Count),
Values = exCols,
Help = "Specify here the new order in which the main columns should appear."
},
new MultiChoiceParam("Numerical columns")
{
Value = ArrayUtils.ConsecutiveInts(numCols.Count),
Values = numCols,
Help = "Specify here the new order in which the numerical columns should appear."
},
new MultiChoiceParam("Multi-numerical columns")
{
Value = ArrayUtils.ConsecutiveInts(multiNumCols.Count),
Values = multiNumCols,
Help = "Specify here the new order in which the numerical columns should appear."
},
new MultiChoiceParam("Categorical columns")
{
Value = ArrayUtils.ConsecutiveInts(catCols.Count),
Values = catCols,
Help = "Specify here the new order in which the categorical columns should appear."
},
new MultiChoiceParam("Text columns")
{
Value = ArrayUtils.ConsecutiveInts(textCols.Count),
Values = textCols,
Help = "Specify here the new order in which the text columns should appear."
}
}));
}
public Parameters GetParameters(IMatrixData mdata, ref string errorString)
{
IList <string> values = mdata.MultiNumericColumnNames;
int[] sel = ArrayUtils.ConsecutiveInts(values.Count);
return
(new Parameters(new Parameter[] {
new MultiChoiceParam("Operation")
{
Values = names,
Help = "How should the numbers in a cell of the multi-numeric columns be transformed to a single number?"
},
new MultiChoiceParam("Columns")
{
Values = values,
Value = sel,
Help = "Select here the multi-numeric colums that should be converted to numeric columns."
}
}));
}
public void LoadData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables, ref IDocumentData[] documents,
ProcessInfo processInfo)
{
var remoteExe = GetExectuable(param);
var paramFile = Path.GetTempFileName();
param.ToFile(paramFile);
var outFile = Path.GetTempFileName();
var codeFile = GetCodeFile(param);
var args = $"{codeFile} {paramFile} {outFile}";
Debug.WriteLine($"executing > {remoteExe} {args}");
if (Utils.RunProcess(remoteExe, args, processInfo.Status, out string processInfoErrString) != 0)
{
processInfo.ErrString = processInfoErrString;
return;
}
;
PerseusUtils.ReadMatrixFromFile(mdata, processInfo, outFile, '\t');
}
private static IMatrixData GetResult(IMatrixData mdata1, IMatrixData mdata2,
Parameters parameters, IList <int[]> indexMap)
{
IMatrixData result = (IMatrixData)mdata1.Clone();
SetAnnotationRows(result, mdata1, mdata2);
bool indicator = parameters.GetParam <bool>("Indicator").Value;
if (indicator)
{
string[][] indicatorCol = new string[indexMap.Count][];
for (int i = 0; i < indexMap.Count; i++)
{
indicatorCol[i] = indexMap[i].Length > 0 ? new[] { "+" } : new string[0];
}
result.AddCategoryColumn(mdata2.Name, "", indicatorCol);
}
result.Origin = "Combination";
return(result);
}
public static void FilterColumns(IMatrixData mdata, Parameters parameters, int[] cols)
{
bool reduceMatrix = GetReduceMatrix(parameters);
if (reduceMatrix)
{
mdata.ExtractColumns(cols);
}
else
{
Array.Sort(cols);
string[][] row = new string[mdata.ColumnCount][];
for (int i = 0; i < row.Length; i++)
{
bool contains = Array.BinarySearch(cols, i) >= 0;
row[i] = contains ? new[] { "Keep" } : new[] { "Discard" };
}
mdata.AddCategoryRow("Filter", "", row);
}
}
public static void FilterRows(IMatrixData mdata, Parameters parameters, int[] rows)
{
bool reduceMatrix = GetReduceMatrix(parameters);
if (reduceMatrix)
{
mdata.ExtractRows(rows);
}
else
{
Array.Sort(rows);
string[][] col = new string[mdata.RowCount][];
for (int i = 0; i < col.Length; i++)
{
bool contains = Array.BinarySearch(rows, i) >= 0;
col[i] = contains ? new[] { "Keep" } : new[] { "Discard" };
}
mdata.AddCategoryColumn("Filter", "", col);
}
}
public void ProcessData(IMatrixData data, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] numColInds = param.GetParam <int[]>("Numerical rows").Value;
int[] multiNumColInds = param.GetParam <int[]>("Multi-numerical rows").Value;
int[] catColInds = param.GetParam <int[]>("Categorical rows").Value;
int[] textColInds = param.GetParam <int[]>("Text rows").Value;
data.NumericRows = ArrayUtils.SubList(data.NumericRows, numColInds);
data.NumericRowNames = ArrayUtils.SubList(data.NumericRowNames, numColInds);
data.NumericRowDescriptions = ArrayUtils.SubList(data.NumericRowDescriptions, numColInds);
data.MultiNumericRows = ArrayUtils.SubList(data.MultiNumericRows, multiNumColInds);
data.MultiNumericRowNames = ArrayUtils.SubList(data.MultiNumericRowNames, multiNumColInds);
data.MultiNumericRowDescriptions = ArrayUtils.SubList(data.MultiNumericRowDescriptions, multiNumColInds);
data.CategoryRows = PerseusPluginUtils.GetCategoryRows(data, catColInds);
data.CategoryRowNames = ArrayUtils.SubList(data.CategoryRowNames, catColInds);
data.CategoryRowDescriptions = ArrayUtils.SubList(data.CategoryRowDescriptions, catColInds);
data.StringRows = ArrayUtils.SubList(data.StringRows, textColInds);
data.StringRowNames = ArrayUtils.SubList(data.StringRowNames, textColInds);
data.StringRowDescriptions = ArrayUtils.SubList(data.StringRowDescriptions, textColInds);
}
public IAnalysisResult AnalyzeData(IMatrixData mdata, Parameters param, ProcessInfo processInfo)
{
var remoteExe = param.GetParam <string>(InterpreterLabel).Value;
var inFile = Path.GetTempFileName();
PerseusUtils.WriteMatrixToFile(mdata, inFile);
var paramFile = Path.GetTempFileName();
param.ToFile(paramFile);
var outFile = Path.GetTempFileName();
var codeFile = GetCodeFile(param);
var args = $"{codeFile} {paramFile} {inFile} {outFile}";
if (Utils.RunProcess(remoteExe, args, processInfo.Status, out string errorString) != 0)
{
processInfo.ErrString = errorString;
return(null);
}
return(GenerateResult(outFile, mdata, processInfo));
}
public void TestSmallExample3()
{
Parameter <int[]> mainColParam = parameters.GetParam <int[]>("Copy main columns");
mainColParam.Value = new[] { 0 };
SingleChoiceParam matchColParam1 = (SingleChoiceParam)parameters.GetParam <int>("Matching column in table 1");
CollectionAssert.AreEqual(new [] { "pep_id", "pep_Protein group IDs", "pep_Intensity" }, matchColParam1.Values.ToArray());
matchColParam1.Value = 1;
Assert.AreEqual("pep_Protein group IDs", matchColParam1.StringValue);
IMatrixData[] supplTables = null;
IDocumentData[] documents = null;
IMatrixData matched = matching.ProcessData(new[] { peptides, proteinMain }, parameters, ref supplTables, ref documents, BaseTest.CreateProcessInfo());
CollectionAssert.AreEqual(new [] { "pep_MS/MS Count", "prot_LFQ intensity", "pep_id", "pep_Protein group IDs", "pep_Intensity" },
matched.ColumnNames.Concat(matched.StringColumnNames).Concat(matched.NumericColumnNames).ToArray());
Assert.AreEqual(2, matched.RowCount);
Assert.AreEqual(2, matched.ColumnCount);
Assert.AreEqual(1, matched.NumericColumnCount);
}
private static void Calc2(int j, IMatrixData data, IList <double> means, IList <double> stddevs, bool report,
bool median)
{
double[] vals = new double[data.RowCount];
for (int i = 0; i < data.RowCount; i++)
{
vals[i] = data.Values.Get(i, j);
}
double mean = ArrayUtils.MeanAndStddev(vals, out double stddev, median);
for (int i = 0; i < data.RowCount; i++)
{
data.Values.Set(i, j, ((data.Values.Get(i, j) - mean) / stddev));
}
if (report)
{
means[j] = mean;
stddevs[j] = stddev;
}
}
public Parameters GetCreateParameters(IMatrixData mdata)
{
List <Parameter> par = new List <Parameter> {
new StringParam("Row name")
{
Value = "Quantity1", Help = "Name of the new numerical annotation row."
}
};
for (int i = 0; i < mdata.ColumnNames.Count; i++)
{
string t = mdata.ColumnNames[i];
string help = "Specify a numerical value for the column '" + t + "'.";
par.Add(new DoubleParam(t, i + 1.0)
{
Help = help
});
}
return(new Parameters(par));
}
//public string HelpOutput { get; }
//public string[] HelpSupplTables { get; }
//public int NumSupplTables { get; }
//public string[] HelpDocuments { get; }
//public int NumDocuments { get; }
public override Parameters GetParameters(IMatrixData mdata, ref string errString)
{
Parameters parameters = new Parameters
(
PECAParameters.GetWorkingDir()
);
parameters.AddParameterGroup(PECAParameters.GetAboutData(), "About Data", false);
parameters.AddParameterGroup(PECAParameters.GetFeatures(), "Features", false);
parameters.AddParameterGroup(PECAParameters.GetModule(), PECAParameters.network + " Info", false);
parameters.AddParameterGroup(PECAParameters.SelectConditionalData(mdata), "Select Data", true);
parameters.AddParameterGroup(PECAParameters.GetMCMCParams(), "MCMC Parameters", false);
return(parameters);
}
public string[] ExtractGroup(IMatrixData mdata, ParameterWithSubParams <int> p, ProcessInfo processInfo,
string value, int colInd)
{
string[] errors = new string[0];
Parameter <int[]> mcp = p.GetSubParameters().GetParam <int[]>(value);
int[] inds = mcp.Value;
if (inds.Length == 0)
{
return(errors);
}
string[] values = new string[inds.Length];
string[] groupids = new string[inds.Length];
string[] v = mdata.GetCategoryRowValuesAt(colInd);
for (int i = 0; i < values.Length; i++)
{
groupids[i] = v[inds[i]];
}
return(groupids);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
ParameterWithSubParams <int> scwsp = param.GetParamWithSubParams <int>("Action");
Parameters spar = scwsp.GetSubParameters();
switch (scwsp.Value)
{
case 0:
ProcessDataCreate(mdata, spar);
break;
case 1:
ProcessDataCreateFromGoupNames(mdata, spar, processInfo);
break;
case 2:
ProcessDataEdit(mdata, spar);
break;
case 3:
ProcessDataRename(mdata, spar);
break;
case 4:
ProcessDataDelete(mdata, spar);
break;
case 5:
ProcessDataWriteTemplateFile(mdata, spar);
break;
case 6:
string err = ProcessDataReadFromFile(mdata, spar);
if (err != null)
{
processInfo.ErrString = err;
}
break;
}
}
public void ProcessData(IMatrixData data, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
bool falseAreIndicated = param.GetParam <int>("Indicated are").Value == 0;
int catCol = param.GetParam <int>("In column").Value;
string word = param.GetParam <string>("Indicator").Value;
int[] scoreColumns = param.GetParam <int[]>("Scores").Value;
if (scoreColumns.Length == 0)
{
processInfo.ErrString = "Please specify at least one column with scores.";
return;
}
bool largeIsGood = param.GetParam <bool>("Large values are good").Value;
int[] showColumns = param.GetParam <int[]>("Display quantity").Value;
if (showColumns.Length == 0)
{
processInfo.ErrString = "Please select at least one quantity to display";
return;
}
bool[] indCol = GetIndicatorColumn(falseAreIndicated, catCol, word, data);
List <string> expColNames = new List <string>();
List <float[]> expCols = new List <float[]>();
foreach (int scoreColumn in scoreColumns)
{
double[] vals = scoreColumn < data.NumericColumnCount
? data.NumericColumns[scoreColumn]
: ArrayUtils.ToDoubles(data.Values.GetColumn(scoreColumn - data.NumericColumnCount));
string name = scoreColumn < data.NumericColumnCount
? data.NumericColumnNames[scoreColumn] : data.ColumnNames[scoreColumn - data.NumericColumnCount];
int[] order = GetOrder(vals, largeIsGood);
CalcCurve(ArrayUtils.SubArray(indCol, order), showColumns, name, expCols, expColNames);
}
float[,] expData = ToMatrix(expCols);
data.ColumnNames = expColNames;
data.Values.Set(expData);
data.SetAnnotationColumns(new List <string>(), new List <string[]>(), new List <string>(),
new List <string[][]>(), new List <string>(), new List <double[]>(), new List <string>(), new List <double[][]>());
}
public void RunDESeq2(Dictionary <string, int> samples, List <string> pairs1, List <string> pairs2,
IMatrixData mdata, REngine engine, ParameterWithSubParams <bool> fdrValid,
ParameterWithSubParams <bool> pValid, ParameterWithSubParams <bool> lfcValid)
{
engine.Evaluate("library(DESeq2)");
engine.Evaluate("counts <- read.delim('Count_table.txt')");
string keys = "", values = "";
foreach (KeyValuePair <string, int> entry in samples)
{
if (keys.Length == 0)
{
keys = "'" + entry.Key + "'";
values = entry.Value.ToString();
}
else
{
keys = keys + ", " + "'" + entry.Key + "'";
values = values + ", " + entry.Value.ToString();
}
}
string commandLine = String.Format("sample<-data.frame(groups = rep(c({0}), time = c({1})))", keys, values);
engine.Evaluate(commandLine);
engine.Evaluate("ds<-DESeqDataSetFromMatrix(countData = counts, colData = sample, design = ~groups)");
engine.Evaluate("colnames(ds)<-colnames(counts)");
engine.Evaluate("ds<-DESeq(ds)");
foreach (string pair1 in pairs1)
{
foreach (string pair2 in pairs2)
{
if (pair1 != pair2)
{
commandLine = String.Format("res<-results(ds, c('groups', '{0}', '{1}'))", pair1, pair2);
engine.Evaluate(commandLine);
engine.Evaluate("write.csv(res, file = 'results.csv')");
ExtractDESeq2Results(mdata, pair1, pair2, fdrValid, pValid, lfcValid);
}
}
}
}
