public override object[] GetRowData(long row)
{
List <object> rowData = new List <object>();
for (int i = 0; i < mdata.ColumnCount; i++)
{
rowData.Add(NumUtils.RoundSignificantDigits(mdata.Values.Get((int)row, i), 6));
}
for (int i = 0; i < mdata.CategoryColumnCount; i++)
{
rowData.Add(StringUtils.Concat(";", mdata.GetCategoryColumnEntryAt(i, (int)row) ?? new string[0]));
}
for (int i = 0; i < mdata.NumericColumnCount; i++)
{
rowData.Add(NumUtils.RoundSignificantDigits(mdata.NumericColumns[i][row], 6));
}
for (int i = 0; i < mdata.StringColumnCount; i++)
{
rowData.Add(mdata.StringColumns[i][row]);
}
for (int i = 0; i < mdata.MultiNumericColumnCount; i++)
{
rowData.Add(StringUtils.Concat(";", mdata.MultiNumericColumns[i][row] ?? new double[0]));
}
return(rowData.ToArray());
}
private string GetRowName(int ind, int nameColumnIndex, bool cutNames)
{
if (nameColumnIndex < 0)
{
return("");
}
if (nameColumnIndex < mdata.StringColumnCount)
{
string[] x = mdata.StringColumns[nameColumnIndex];
if (ind >= 0 && ind < x.Length)
{
if (cutNames && x[ind].Contains(";"))
{
return(x[ind].Substring(0, x[ind].IndexOf(';')));
}
return(x[ind]);
}
}
int indi = nameColumnIndex - mdata.StringColumnCount;
if (indi < mdata.CategoryColumnCount && indi >= 0)
{
if (ind >= 0 && ind < mdata.RowCount)
{
string[] w = mdata.GetCategoryColumnEntryAt(indi, ind);
if (cutNames)
{
return(w.Length > 0 ? w[0] : "");
}
return(StringUtils.Concat(";", w));
}
}
return("");
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
ParameterWithSubParams <int> p = param.GetParamWithSubParams <int>("Column");
int colInd = p.Value;
if (colInd < 0)
{
processInfo.ErrString = "No categorical columns available.";
return;
}
Parameter <int[]> mcp = p.GetSubParameters().GetParam <int[]>("Values");
int[] inds = mcp.Value;
if (inds.Length == 0)
{
processInfo.ErrString = "Please select at least one term for filtering.";
return;
}
string[] values = new string[inds.Length];
string[] v = mdata.GetCategoryColumnValuesAt(colInd);
for (int i = 0; i < values.Length; i++)
{
values[i] = v[inds[i]];
}
HashSet <string> value = new HashSet <string>(values);
bool remove = param.GetParam <int>("Mode").Value == 0;
List <int> valids = new List <int>();
List <int> notvalids = new List <int>();
for (int i = 0; i < mdata.RowCount; i++)
{
bool valid = true;
foreach (string w in mdata.GetCategoryColumnEntryAt(colInd, i))
{
if (value.Contains(w))
{
valid = false;
break;
}
}
if (valid && remove || !valid && !remove)
{
valids.Add(i);
}
else if (!valid)
{
notvalids.Add(i);
}
}
if (param.GetParam <int>("Filter mode").Value == 2)
{
supplTables = new[] { PerseusPluginUtils.CreateSupplTabSplit(mdata, notvalids.ToArray()) };
}
PerseusPluginUtils.FilterRowsNew(mdata, param, valids.ToArray());
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
string colName = param.GetParam<string>("Name of new column").Value;
int[] columns = param.GetParam<int[]>("Categories").Value;
bool inverse = param.GetParam<bool>("Inverse").Value;
int[] catCols;
int[] stringCols;
Split(columns, out catCols, out stringCols, mdata.CategoryColumnCount);
string[] word1 = param.GetParam<string[]>("Search terms").Value;
if (word1.Length == 0){
processInfo.ErrString = "Please specify one or more search terms.";
return;
}
if (string.IsNullOrEmpty(colName)){
colName = word1[0];
}
string[] word = new string[word1.Length];
for (int i = 0; i < word.Length; i++){
word[i] = word1[i].ToLower().Trim();
}
bool[] indicator = new bool[mdata.RowCount];
foreach (int col in catCols){
for (int i = 0; i < mdata.RowCount; i++){
foreach (string s in mdata.GetCategoryColumnEntryAt(col, i)){
foreach (string s1 in word){
if (s.ToLower().Contains(s1)){
indicator[i] = true;
break;
}
}
}
}
}
foreach (string[] txt in stringCols.Select(col => mdata.StringColumns[col])){
for (int i = 0; i < txt.Length; i++){
string s = txt[i];
foreach (string s1 in word){
if (s.ToLower().Contains(s1)){
indicator[i] = true;
break;
}
}
}
}
string[][] newCol = new string[indicator.Length][];
for (int i = 0; i < newCol.Length; i++){
bool yes = inverse ? !indicator[i] : indicator[i];
newCol[i] = yes ? new[]{"+"} : new string[0];
}
mdata.AddCategoryColumn(colName, "", newCol);
}
public static int[] GetIndicesOf(IMatrixData data, string categoryName, HashSet <string> values)
{
int index = GetIndexOf(data, categoryName);
List <int> result = new List <int>();
for (int i = 0; i < data.RowCount; i++)
{
string[] s = data.GetCategoryColumnEntryAt(index, i);
foreach (string s1 in s)
{
if (values.Contains(s1))
{
result.Add(i);
break;
}
}
}
return(result.ToArray());
}
public IMatrixData ProcessData(IMatrixData[] inputData, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
IMatrixData mdata1 = inputData[0];
IMatrixData mdata2 = inputData[1];
int nrows1 = mdata1.RowCount;
int nrows2 = mdata2.RowCount;
int nrows = nrows1 + nrows2;
string[] expColNames = SpecialSort(mdata1.ColumnNames, mdata2.ColumnNames, out Dictionary <string, int> dic1, out Dictionary <string, int> dic2);
double[,] ex = new double[nrows, expColNames.Length];
for (int i = 0; i < ex.GetLength(0); i++)
{
for (int j = 0; j < ex.GetLength(1); j++)
{
ex[i, j] = double.NaN;
}
}
for (int i = 0; i < expColNames.Length; i++)
{
if (dic1.ContainsKey(expColNames[i]))
{
int ind = dic1[expColNames[i]];
for (int j = 0; j < nrows1; j++)
{
ex[j, i] = mdata1.Values.Get(j, ind);
}
}
if (dic2.ContainsKey(expColNames[i]))
{
int ind = dic2[expColNames[i]];
for (int j = 0; j < nrows2; j++)
{
ex[nrows1 + j, i] = mdata2.Values.Get(j, ind);
}
}
}
string[] numColNames = SpecialSort(mdata1.NumericColumnNames, mdata2.NumericColumnNames, out dic1, out dic2);
List <double[]> numCols = new List <double[]>();
for (int i = 0; i < numColNames.Length; i++)
{
numCols.Add(new double[nrows]);
for (int j = 0; j < nrows; j++)
{
numCols[numCols.Count - 1][j] = double.NaN;
}
}
for (int i = 0; i < numColNames.Length; i++)
{
if (dic1.ContainsKey(numColNames[i]))
{
int ind = dic1[numColNames[i]];
for (int j = 0; j < nrows1; j++)
{
numCols[i][j] = mdata1.NumericColumns[ind][j];
}
}
if (dic2.ContainsKey(numColNames[i]))
{
int ind = dic2[numColNames[i]];
for (int j = 0; j < nrows2; j++)
{
numCols[i][nrows1 + j] = mdata2.NumericColumns[ind][j];
}
}
}
string[] stringColNames = SpecialSort(mdata1.StringColumnNames, mdata2.StringColumnNames, out dic1, out dic2);
List <string[]> stringCols = new List <string[]>();
for (int i = 0; i < stringColNames.Length; i++)
{
stringCols.Add(new string[nrows]);
for (int j = 0; j < nrows; j++)
{
stringCols[stringCols.Count - 1][j] = "";
}
}
for (int i = 0; i < stringColNames.Length; i++)
{
if (dic1.ContainsKey(stringColNames[i]))
{
int ind = dic1[stringColNames[i]];
for (int j = 0; j < nrows1; j++)
{
stringCols[i][j] = mdata1.StringColumns[ind][j];
}
}
if (dic2.ContainsKey(stringColNames[i]))
{
int ind = dic2[stringColNames[i]];
for (int j = 0; j < nrows2; j++)
{
stringCols[i][nrows1 + j] = mdata2.StringColumns[ind][j];
}
}
}
string[] catColNames = SpecialSort(mdata1.CategoryColumnNames, mdata2.CategoryColumnNames, out dic1, out dic2);
List <string[][]> catCols = new List <string[][]>();
for (int i = 0; i < catColNames.Length; i++)
{
catCols.Add(new string[nrows][]);
for (int j = 0; j < nrows; j++)
{
catCols[catCols.Count - 1][j] = new string[0];
}
}
for (int i = 0; i < catColNames.Length; i++)
{
if (dic1.ContainsKey(catColNames[i]))
{
int ind = dic1[catColNames[i]];
for (int j = 0; j < nrows1; j++)
{
catCols[i][j] = mdata1.GetCategoryColumnEntryAt(ind, j);
}
}
if (dic2.ContainsKey(catColNames[i]))
{
int ind = dic2[catColNames[i]];
for (int j = 0; j < nrows2; j++)
{
catCols[i][nrows1 + j] = mdata2.GetCategoryColumnEntryAt(ind, j);
}
}
}
string[] multiNumColNames = SpecialSort(mdata1.MultiNumericColumnNames, mdata2.MultiNumericColumnNames, out dic1,
out dic2);
List <double[][]> multiNumCols = new List <double[][]>();
for (int i = 0; i < multiNumColNames.Length; i++)
{
multiNumCols.Add(new double[nrows][]);
for (int j = 0; j < nrows; j++)
{
multiNumCols[multiNumCols.Count - 1][j] = new double[0];
}
}
for (int i = 0; i < multiNumColNames.Length; i++)
{
if (dic1.ContainsKey(multiNumColNames[i]))
{
int ind = dic1[multiNumColNames[i]];
for (int j = 0; j < nrows1; j++)
{
multiNumCols[i][j] = mdata1.MultiNumericColumns[ind][j];
}
}
if (dic2.ContainsKey(multiNumColNames[i]))
{
int ind = dic2[multiNumColNames[i]];
for (int j = 0; j < nrows2; j++)
{
multiNumCols[i][nrows1 + j] = mdata2.MultiNumericColumns[ind][j];
}
}
}
IMatrixData result = PerseusFactory.CreateMatrixData(ex, expColNames.ToList());
result.ColumnDescriptions = result.ColumnNames;
result.NumericColumnNames = new List <string>(numColNames);
result.NumericColumnDescriptions = result.NumericColumnNames;
result.NumericColumns = numCols;
result.StringColumnNames = new List <string>(stringColNames);
result.StringColumnDescriptions = result.StringColumnDescriptions;
result.StringColumns = stringCols;
result.CategoryColumnNames = new List <string>(catColNames);
result.CategoryColumnDescriptions = result.CategoryColumnNames;
result.CategoryColumns = catCols;
result.MultiNumericColumnNames = new List <string>(multiNumColNames);
result.MultiNumericColumnDescriptions = result.MultiNumericColumnNames;
result.MultiNumericColumns = multiNumCols;
return(result);
}
/// <summary>
/// Write matrix to file with tab separation
/// </summary>
/// <param name="data"></param>
/// <param name="filename"></param>
/// <param name="addtlMatrices">if true numbers are converted to triples <code>value;imputed;quality</code></param>
public static void WriteMatrixToFile(IMatrixData data, string filename, bool addtlMatrices=false)
{
using (var writer = new StreamWriter(filename))
{
List<string> words = new List<string>();
for (int i = 0; i < data.ColumnCount; i++)
{
words.Add(data.ColumnNames[i]);
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
words.Add(data.CategoryColumnNames[i]);
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add(data.NumericColumnNames[i]);
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add(data.StringColumnNames[i]);
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
words.Add(data.MultiNumericColumnNames[i]);
}
writer.WriteLine(StringUtils.Concat("\t", words));
if (HasAnyDescription(data))
{
words = new List<string>();
for (int i = 0; i < data.ColumnCount; i++)
{
words.Add(data.ColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
words.Add(data.CategoryColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add(data.NumericColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add(data.StringColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
words.Add(data.MultiNumericColumnDescriptions[i] ?? "");
}
writer.WriteLine("#!{Description}" + StringUtils.Concat("\t", words));
}
words = new List<string>();
for (int i = 0; i < data.ColumnCount; i++)
{
words.Add("E");
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
words.Add("C");
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add("N");
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add("T");
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
words.Add("M");
}
writer.WriteLine("#!{Type}" + StringUtils.Concat("\t", words));
for (int i = 0; i < data.NumericRowCount; i++)
{
words = new List<string>();
for (int j = 0; j < data.ColumnCount; j++)
{
words.Add("" + data.NumericRows[i][j]);
}
for (int j = 0; j < data.CategoryColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.NumericColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.StringColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.MultiNumericColumnCount; j++)
{
words.Add("");
}
writer.WriteLine("#!{N:" + data.NumericRowNames[i] + "}" + StringUtils.Concat("\t", words));
}
for (int i = 0; i < data.CategoryRowCount; i++)
{
words = new List<string>();
for (int j = 0; j < data.ColumnCount; j++)
{
string[] s = data.GetCategoryRowAt(i)[j];
words.Add(s.Length == 0 ? "" : StringUtils.Concat(";", s));
}
for (int j = 0; j < data.CategoryColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.NumericColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.StringColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.MultiNumericColumnCount; j++)
{
words.Add("");
}
writer.WriteLine("#!{C:" + data.CategoryRowNames[i] + "}" + StringUtils.Concat("\t", words));
}
for (int j = 0; j < data.RowCount; j++)
{
words = new List<string>();
for (int i = 0; i < data.ColumnCount; i++)
{
string s1 = "" + data.Values.Get(j, i);
if (addtlMatrices)
{
s1 += ";" + data.IsImputed[j, i] + ";" + data.Quality.Get(j, i);
}
words.Add(s1);
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
string[] q = data.GetCategoryColumnEntryAt(i, j) ?? new string[0];
words.Add((q.Length > 0 ? StringUtils.Concat(";", q) : ""));
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add("" + data.NumericColumns[i][j]);
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add(data.StringColumns[i][j]);
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
double[] q = data.MultiNumericColumns[i][j];
words.Add((q.Length > 0 ? StringUtils.Concat(";", q) : ""));
}
string s = StringUtils.Concat("\t", words);
writer.WriteLine(s);
}
}
}
public IMatrixData ProcessData(IMatrixData[] inputData, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
IMatrixData mdata1 = inputData[0];
IMatrixData mdata2 = inputData[1];
string[] header1 = new string[mdata1.RowCount];
for (int i = 0; i < mdata1.RowCount; i++)
{
header1[i] = mdata1.Name;
}
string[] header2 = new string[mdata2.RowCount];
for (int i = 0; i < mdata2.RowCount; i++)
{
header2[i] = mdata2.Name;
}
int nrows1 = mdata1.RowCount;
int nrows2 = mdata2.RowCount;
int nrows = nrows1 + nrows2;
string[] expColNames = SpecialSort(mdata1.ColumnNames, mdata2.ColumnNames, out Dictionary <string, int> dic1, out Dictionary <string, int> dic2);
double[,] ex = new double[nrows, expColNames.Length];
for (int i = 0; i < ex.GetLength(0); i++)
{
for (int j = 0; j < ex.GetLength(1); j++)
{
ex[i, j] = double.NaN;
}
}
for (int i = 0; i < expColNames.Length; i++)
{
if (dic1.ContainsKey(expColNames[i]))
{
int ind = dic1[expColNames[i]];
for (int j = 0; j < nrows1; j++)
{
ex[j, i] = mdata1.Values.Get(j, ind);
}
}
if (dic2.ContainsKey(expColNames[i]))
{
int ind = dic2[expColNames[i]];
for (int j = 0; j < nrows2; j++)
{
ex[nrows1 + j, i] = mdata2.Values.Get(j, ind);
}
}
}
string[] numColNames = SpecialSort(mdata1.NumericColumnNames, mdata2.NumericColumnNames, out dic1, out dic2);
List <double[]> numCols = new List <double[]>();
for (int i = 0; i < numColNames.Length; i++)
{
numCols.Add(new double[nrows]);
for (int j = 0; j < nrows; j++)
{
numCols[numCols.Count - 1][j] = double.NaN;
}
}
for (int i = 0; i < numColNames.Length; i++)
{
if (dic1.ContainsKey(numColNames[i]))
{
int ind = dic1[numColNames[i]];
for (int j = 0; j < nrows1; j++)
{
numCols[i][j] = mdata1.NumericColumns[ind][j];
}
}
if (dic2.ContainsKey(numColNames[i]))
{
int ind = dic2[numColNames[i]];
for (int j = 0; j < nrows2; j++)
{
numCols[i][nrows1 + j] = mdata2.NumericColumns[ind][j];
}
}
}
string[] stringColNames = SpecialSort(mdata1.StringColumnNames, mdata2.StringColumnNames, out dic1, out dic2);
List <string[]> stringCols = new List <string[]>();
for (int i = 0; i < stringColNames.Length; i++)
{
stringCols.Add(new string[nrows]);
for (int j = 0; j < nrows; j++)
{
stringCols[stringCols.Count - 1][j] = "";
}
}
for (int i = 0; i < stringColNames.Length; i++)
{
if (dic1.ContainsKey(stringColNames[i]))
{
int ind = dic1[stringColNames[i]];
for (int j = 0; j < nrows1; j++)
{
stringCols[i][j] = mdata1.StringColumns[ind][j];
}
}
if (dic2.ContainsKey(stringColNames[i]))
{
int ind = dic2[stringColNames[i]];
for (int j = 0; j < nrows2; j++)
{
stringCols[i][nrows1 + j] = mdata2.StringColumns[ind][j];
}
}
}
string[] catColNames = SpecialSort(mdata1.CategoryColumnNames, mdata2.CategoryColumnNames, out dic1, out dic2);
List <string[][]> catCols = new List <string[][]>();
for (int i = 0; i < catColNames.Length; i++)
{
catCols.Add(new string[nrows][]);
for (int j = 0; j < nrows; j++)
{
catCols[catCols.Count - 1][j] = new string[0];
}
}
for (int i = 0; i < catColNames.Length; i++)
{
if (dic1.ContainsKey(catColNames[i]))
{
int ind = dic1[catColNames[i]];
for (int j = 0; j < nrows1; j++)
{
catCols[i][j] = mdata1.GetCategoryColumnEntryAt(ind, j);
}
}
if (dic2.ContainsKey(catColNames[i]))
{
int ind = dic2[catColNames[i]];
for (int j = 0; j < nrows2; j++)
{
catCols[i][nrows1 + j] = mdata2.GetCategoryColumnEntryAt(ind, j);
}
}
}
string[] multiNumColNames = SpecialSort(mdata1.MultiNumericColumnNames, mdata2.MultiNumericColumnNames, out dic1,
out dic2);
List <double[][]> multiNumCols = new List <double[][]>();
for (int i = 0; i < multiNumColNames.Length; i++)
{
multiNumCols.Add(new double[nrows][]);
for (int j = 0; j < nrows; j++)
{
multiNumCols[multiNumCols.Count - 1][j] = new double[0];
}
}
for (int i = 0; i < multiNumColNames.Length; i++)
{
if (dic1.ContainsKey(multiNumColNames[i]))
{
int ind = dic1[multiNumColNames[i]];
for (int j = 0; j < nrows1; j++)
{
multiNumCols[i][j] = mdata1.MultiNumericColumns[ind][j];
}
}
if (dic2.ContainsKey(multiNumColNames[i]))
{
int ind = dic2[multiNumColNames[i]];
for (int j = 0; j < nrows2; j++)
{
multiNumCols[i][nrows1 + j] = mdata2.MultiNumericColumns[ind][j];
}
}
}
string MatrixName = "Matrix Name";
string MatrixDescription = "Description";
string[] listnames = header1.Concat(header2).ToArray();
// string[][] resultarray = catlistnames.Select(x => x.ToArray()).ToArray();
//IMPORTANT!!!!! TODO: check if the name of the matrix if changed
IMatrixData result = PerseusFactory.CreateMatrixData(ex, expColNames.ToList());
result.NumericColumnNames = new List <string>(numColNames);
result.NumericColumnDescriptions = result.NumericColumnNames;
result.NumericColumns = numCols;
result.StringColumnNames = new List <string>(stringColNames);
result.StringColumns = stringCols;
result.CategoryColumnNames = new List <string>(catColNames);
result.CategoryColumnDescriptions = result.CategoryColumnNames;
result.CategoryColumns = catCols;
result.MultiNumericColumnNames = new List <string>(multiNumColNames);
result.MultiNumericColumnDescriptions = result.MultiNumericColumnNames;
result.MultiNumericColumns = multiNumCols;
HashSet <string> taken = new HashSet <string>(result.StringColumnNames);
result.AddStringColumn(MatrixName, MatrixName, listnames);
taken.Add(MatrixName);
return(result);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
List <string> expNames = mdata.ColumnNames;
string errorString = null;
int[,] colInds = SortNumberedNames(expNames, mdata.ColumnDescriptions, maxInd, ref errorString,
out string[] allSuffixes, out string[] allPrefixes, out List <string> allDescriptions);
if (errorString != null)
{
processInfo.ErrString = errorString;
return;
}
int[] normalIndices = ArrayUtils.Complement(To1DArray(colInds), expNames.Count);
normalIndices = FilterExpressionColIndices(normalIndices, mdata.ColumnNames, allPrefixes);
int[] validNumCols = GetValidNumCols(mdata.NumericColumnNames, allPrefixes);
int nrows = mdata.RowCount * allSuffixes.Length;
int ncols = normalIndices.Length + allPrefixes.Length;
double[,] data = new double[nrows, ncols];
double[,] quality = new double[nrows, ncols];
bool[,] imputed = new bool[nrows, ncols];
List <double[]> numCols = new List <double[]>();
for (int i = 0; i < validNumCols.Length; i++)
{
numCols.Add(new double[nrows]);
}
List <string[]> stringCols = new List <string[]>();
for (int i = 0; i < mdata.StringColumnCount + 1; i++)
{
stringCols.Add(new string[nrows]);
}
List <string[][]> catCols = new List <string[][]>();
for (int i = 0; i < mdata.CategoryColumnCount + 1; i++)
{
catCols.Add(new string[nrows][]);
}
List <double[][]> multiNumCols = new List <double[][]>();
for (int i = 0; i < mdata.MultiNumericColumnCount; i++)
{
multiNumCols.Add(new double[nrows][]);
}
List <string> expColNames = new List <string>();
List <string> expColDescriptions = new List <string>();
foreach (int t in normalIndices)
{
expColNames.Add(expNames[t]);
expColDescriptions.Add(mdata.ColumnDescriptions[t]);
}
foreach (Tuple <string, string> t in allPrefixes.Zip(allDescriptions, Tuple.Create))
{
expColNames.Add(t.Item1);
expColDescriptions.Add(t.Item2);
}
int count = 0;
for (int i = 0; i < allSuffixes.Length; i++)
{
for (int j = 0; j < mdata.RowCount; j++)
{
count++;
int rowInd = i * mdata.RowCount + j;
for (int k = 0; k < normalIndices.Length; k++)
{
data[rowInd, k] = mdata.Values.Get(j, normalIndices[k]);
quality[rowInd, k] = mdata.Quality.Get(j, normalIndices[k]);
imputed[rowInd, k] = mdata.IsImputed[j, normalIndices[k]];
}
for (int k = 0; k < allPrefixes.Length; k++)
{
data[rowInd, normalIndices.Length + k] = mdata.Values.Get(j, colInds[k, i]);
quality[rowInd, normalIndices.Length + k] = mdata.Quality.Get(j, colInds[k, i]);
imputed[rowInd, normalIndices.Length + k] = mdata.IsImputed[j, colInds[k, i]];
}
for (int k = 0; k < validNumCols.Length; k++)
{
numCols[k][rowInd] = mdata.NumericColumns[validNumCols[k]][j];
}
for (int k = 0; k < mdata.StringColumnCount; k++)
{
stringCols[k][rowInd] = mdata.StringColumns[k][j];
}
for (int k = 0; k < mdata.CategoryColumnCount; k++)
{
catCols[k][rowInd] = mdata.GetCategoryColumnEntryAt(k, j);
}
for (int k = 0; k < mdata.MultiNumericColumnCount; k++)
{
multiNumCols[k][rowInd] = mdata.MultiNumericColumns[k][j];
}
catCols[mdata.CategoryColumnCount][rowInd] = new[] { allSuffixes[i] };
stringCols[stringCols.Count - 1][count - 1] = "UID" + count;
}
}
string[] catColNames = ArrayUtils.Concat(mdata.CategoryColumnNames, new[] { "Multiplicity" });
mdata.ColumnNames = expColNames;
mdata.ColumnDescriptions = expColDescriptions;
mdata.Values.Set(data);
mdata.Quality.Set(quality);
mdata.IsImputed.Set(imputed);
mdata.SetAnnotationColumns(new List <string>(ArrayUtils.Concat(mdata.StringColumnNames, new[] { "Unique identifier" })),
stringCols, new List <string>(catColNames), catCols, ArrayUtils.SubList(mdata.NumericColumnNames, validNumCols),
numCols, mdata.MultiNumericColumnNames, multiNumCols);
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] multiNumCols = param1.GetParam <int[]>("Multi-numeric columns").Value;
Array.Sort(multiNumCols);
int[] stringCols = param1.GetParam <int[]>("Text columns").Value;
Array.Sort(stringCols);
HashSet <int> multinumCols2 = new HashSet <int>(multiNumCols);
HashSet <int> stringCols2 = new HashSet <int>(stringCols);
if (multiNumCols.Length + stringCols.Length == 0)
{
processInfo.ErrString = "Please select some columns.";
return;
}
int rowCount = GetNewRowCount(mdata, multiNumCols, stringCols);
float[,] expVals = new float[rowCount, mdata.ColumnCount];
List <string[]> stringC = new List <string[]>();
for (int i = 0; i < mdata.StringColumnCount; i++)
{
stringC.Add(new string[rowCount]);
}
List <double[]> numC = new List <double[]>();
for (int i = 0; i < mdata.NumericColumnCount; i++)
{
numC.Add(new double[rowCount]);
}
List <string[][]> catC = new List <string[][]>();
for (int i = 0; i < mdata.CategoryColumnCount; i++)
{
catC.Add(new string[rowCount][]);
}
List <double[][]> multiNumC = new List <double[][]>();
for (int i = 0; i < mdata.MultiNumericColumnCount; i++)
{
multiNumC.Add(new double[rowCount][]);
}
int count = 0;
for (int i = 0; i < mdata.RowCount; i++)
{
string err;
int entryCount = GetEntryCount(i, mdata, multiNumCols, stringCols, out err);
if (err != null)
{
processInfo.ErrString = err;
return;
}
bool empty = entryCount == 0;
entryCount = Math.Max(entryCount, 1);
for (int j = 0; j < entryCount; j++)
{
for (int k = 0; k < mdata.ColumnCount; k++)
{
expVals[count + j, k] = mdata.Values[i, k];
}
for (int k = 0; k < mdata.NumericColumnCount; k++)
{
numC[k][count + j] = mdata.NumericColumns[k][i];
}
for (int k = 0; k < mdata.CategoryColumnCount; k++)
{
catC[k][count + j] = mdata.GetCategoryColumnEntryAt(k, i);
}
}
for (int k = 0; k < mdata.MultiNumericColumnCount; k++)
{
if (multinumCols2.Contains(k))
{
if (empty)
{
multiNumC[k][count] = new double[0];
}
else
{
double[] vals = mdata.MultiNumericColumns[k][i];
for (int j = 0; j < entryCount; j++)
{
multiNumC[k][count + j] = new[] { vals[j] };
}
}
}
else
{
for (int j = 0; j < entryCount; j++)
{
multiNumC[k][count + j] = mdata.MultiNumericColumns[k][i];
}
}
}
for (int k = 0; k < mdata.StringColumnCount; k++)
{
if (stringCols2.Contains(k))
{
if (empty)
{
stringC[k][count] = "";
}
else
{
string[] vals = mdata.StringColumns[k][i].Split(';');
for (int j = 0; j < entryCount; j++)
{
stringC[k][count + j] = vals[j];
}
}
}
else
{
for (int j = 0; j < entryCount; j++)
{
stringC[k][count + j] = mdata.StringColumns[k][i];
}
}
}
count += entryCount;
}
int[] multiNumComplement = ArrayUtils.Complement(multiNumCols, mdata.MultiNumericColumnCount);
List <double[][]> toBeTransformed = ArrayUtils.SubList(multiNumC, multiNumCols);
multiNumC = ArrayUtils.SubList(multiNumC, multiNumComplement);
foreach (double[][] d in toBeTransformed)
{
numC.Add(Transform(d));
}
mdata.ColumnNames = mdata.ColumnNames;
mdata.Values.Set(expVals);
mdata.SetAnnotationColumns(mdata.StringColumnNames, stringC, mdata.CategoryColumnNames, catC,
new List <string>(ArrayUtils.Concat(mdata.NumericColumnNames,
ArrayUtils.SubList(mdata.MultiNumericColumnNames, multiNumCols))), numC,
new List <string>(ArrayUtils.SubArray(mdata.MultiNumericColumnNames, multiNumComplement)), multiNumC);
}
public void ProcessData(IMatrixData mdata, Parameters param1, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
int[] multiNumCols = param1.GetParam<int[]>("Multi-numeric columns").Value;
Array.Sort(multiNumCols);
int[] stringCols = param1.GetParam<int[]>("Text columns").Value;
Array.Sort(stringCols);
HashSet<int> multinumCols2 = new HashSet<int>(multiNumCols);
HashSet<int> stringCols2 = new HashSet<int>(stringCols);
if (multiNumCols.Length + stringCols.Length == 0){
processInfo.ErrString = "Please select some columns.";
return;
}
int rowCount = GetNewRowCount(mdata, multiNumCols, stringCols);
float[,] expVals = new float[rowCount, mdata.ColumnCount];
List<string[]> stringC = new List<string[]>();
for (int i = 0; i < mdata.StringColumnCount; i++){
stringC.Add(new string[rowCount]);
}
List<double[]> numC = new List<double[]>();
for (int i = 0; i < mdata.NumericColumnCount; i++){
numC.Add(new double[rowCount]);
}
List<string[][]> catC = new List<string[][]>();
for (int i = 0; i < mdata.CategoryColumnCount; i++){
catC.Add(new string[rowCount][]);
}
List<double[][]> multiNumC = new List<double[][]>();
for (int i = 0; i < mdata.MultiNumericColumnCount; i++){
multiNumC.Add(new double[rowCount][]);
}
int count = 0;
for (int i = 0; i < mdata.RowCount; i++){
string err;
int entryCount = GetEntryCount(i, mdata, multiNumCols, stringCols, out err);
if (err != null){
processInfo.ErrString = err;
return;
}
bool empty = entryCount == 0;
entryCount = Math.Max(entryCount, 1);
for (int j = 0; j < entryCount; j++){
for (int k = 0; k < mdata.ColumnCount; k++){
expVals[count + j, k] = mdata.Values.Get(i, k);
}
for (int k = 0; k < mdata.NumericColumnCount; k++){
numC[k][count + j] = mdata.NumericColumns[k][i];
}
for (int k = 0; k < mdata.CategoryColumnCount; k++){
catC[k][count + j] = mdata.GetCategoryColumnEntryAt(k, i);
}
}
for (int k = 0; k < mdata.MultiNumericColumnCount; k++){
if (multinumCols2.Contains(k)){
if (empty){
multiNumC[k][count] = new double[0];
} else{
double[] vals = mdata.MultiNumericColumns[k][i];
for (int j = 0; j < entryCount; j++){
multiNumC[k][count + j] = new[]{vals[j]};
}
}
} else{
for (int j = 0; j < entryCount; j++){
multiNumC[k][count + j] = mdata.MultiNumericColumns[k][i];
}
}
}
for (int k = 0; k < mdata.StringColumnCount; k++){
if (stringCols2.Contains(k)){
if (empty){
stringC[k][count] = "";
} else{
string[] vals = mdata.StringColumns[k][i].Split(';');
for (int j = 0; j < entryCount; j++){
stringC[k][count + j] = vals[j];
}
}
} else{
for (int j = 0; j < entryCount; j++){
stringC[k][count + j] = mdata.StringColumns[k][i];
}
}
}
count += entryCount;
}
int[] multiNumComplement = ArrayUtils.Complement(multiNumCols, mdata.MultiNumericColumnCount);
List<double[][]> toBeTransformed = ArrayUtils.SubList(multiNumC, multiNumCols);
multiNumC = ArrayUtils.SubList(multiNumC, multiNumComplement);
foreach (double[][] d in toBeTransformed){
numC.Add(Transform(d));
}
mdata.ColumnNames = mdata.ColumnNames;
mdata.Values.Set(expVals);
mdata.SetAnnotationColumns(mdata.StringColumnNames, stringC, mdata.CategoryColumnNames, catC,
new List<string>(ArrayUtils.Concat(mdata.NumericColumnNames,
ArrayUtils.SubList(mdata.MultiNumericColumnNames, multiNumCols))), numC,
new List<string>(ArrayUtils.SubArray(mdata.MultiNumericColumnNames, multiNumComplement)), multiNumC);
}
public void Export(Parameters parameters, IMatrixData data, ProcessInfo processInfo)
{
string filename = parameters.GetParam <string>("File name").Value;
bool addtlMatrices = parameters.GetParam <bool>("Write quality and imputed matrices").Value;
addtlMatrices = addtlMatrices && data.IsImputed != null && data.Quality != null && data.IsImputed.IsInitialized() &&
data.Quality.IsInitialized();
try{
StreamWriter writer = new StreamWriter(filename);
List <string> words = new List <string>();
for (int i = 0; i < data.ColumnCount; i++)
{
words.Add(data.ColumnNames[i]);
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
words.Add(data.CategoryColumnNames[i]);
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add(data.NumericColumnNames[i]);
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add(data.StringColumnNames[i]);
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
words.Add(data.MultiNumericColumnNames[i]);
}
writer.WriteLine(StringUtils.Concat("\t", words));
if (HasAnyDescription(data))
{
words = new List <string>();
for (int i = 0; i < data.ColumnCount; i++)
{
words.Add(data.ColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
words.Add(data.CategoryColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add(data.NumericColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add(data.StringColumnDescriptions[i] ?? "");
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
words.Add(data.MultiNumericColumnDescriptions[i] ?? "");
}
writer.WriteLine("#!{Description}" + StringUtils.Concat("\t", words));
}
words = new List <string>();
for (int i = 0; i < data.ColumnCount; i++)
{
words.Add("E");
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
words.Add("C");
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add("N");
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add("T");
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
words.Add("M");
}
writer.WriteLine("#!{Type}" + StringUtils.Concat("\t", words));
for (int i = 0; i < data.NumericRowCount; i++)
{
words = new List <string>();
for (int j = 0; j < data.ColumnCount; j++)
{
words.Add("" + data.NumericRows[i][j]);
}
for (int j = 0; j < data.CategoryColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.NumericColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.StringColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.MultiNumericColumnCount; j++)
{
words.Add("");
}
writer.WriteLine("#!{N:" + data.NumericRowNames[i] + "}" + StringUtils.Concat("\t", words));
}
for (int i = 0; i < data.CategoryRowCount; i++)
{
words = new List <string>();
for (int j = 0; j < data.ColumnCount; j++)
{
string[] s = data.GetCategoryRowAt(i)[j];
words.Add(s.Length == 0 ? "" : StringUtils.Concat(";", s));
}
for (int j = 0; j < data.CategoryColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.NumericColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.StringColumnCount; j++)
{
words.Add("");
}
for (int j = 0; j < data.MultiNumericColumnCount; j++)
{
words.Add("");
}
writer.WriteLine("#!{C:" + data.CategoryRowNames[i] + "}" + StringUtils.Concat("\t", words));
}
for (int j = 0; j < data.RowCount; j++)
{
words = new List <string>();
for (int i = 0; i < data.ColumnCount; i++)
{
string s1 = "" + data.Values[j, i];
if (addtlMatrices)
{
s1 += ";" + data.IsImputed[j, i] + ";" + data.Quality[j, i];
}
words.Add(s1);
}
for (int i = 0; i < data.CategoryColumnCount; i++)
{
string[] q = data.GetCategoryColumnEntryAt(i, j) ?? new string[0];
words.Add((q.Length > 0 ? StringUtils.Concat(";", q) : ""));
}
for (int i = 0; i < data.NumericColumnCount; i++)
{
words.Add("" + data.NumericColumns[i][j]);
}
for (int i = 0; i < data.StringColumnCount; i++)
{
words.Add(data.StringColumns[i][j]);
}
for (int i = 0; i < data.MultiNumericColumnCount; i++)
{
double[] q = data.MultiNumericColumns[i][j];
words.Add((q.Length > 0 ? StringUtils.Concat(";", q) : ""));
}
string s = StringUtils.Concat("\t", words);
writer.WriteLine(s);
}
writer.Close();
} catch (Exception e) {
processInfo.ErrString = e.Message;
}
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
string colName = param.GetParam <string>("Name of new column").Value;
int[] columns = param.GetParam <int[]>("Categories").Value;
bool inverse = param.GetParam <bool>("Inverse").Value;
Split(columns, out int[] catCols, out int[] stringCols, mdata.CategoryColumnCount);
string[] word1 = param.GetParam <string[]>("Search terms").Value;
if (word1.Length == 0)
{
processInfo.ErrString = "Please specify one or more search terms.";
return;
}
if (string.IsNullOrEmpty(colName))
{
colName = word1[0];
}
string[] word = new string[word1.Length];
for (int i = 0; i < word.Length; i++)
{
word[i] = word1[i].ToLower().Trim();
}
bool[] indicator = new bool[mdata.RowCount];
foreach (int col in catCols)
{
for (int i = 0; i < mdata.RowCount; i++)
{
foreach (string s in mdata.GetCategoryColumnEntryAt(col, i))
{
foreach (string s1 in word)
{
if (s.ToLower().Contains(s1))
{
indicator[i] = true;
break;
}
}
}
}
}
foreach (string[] txt in stringCols.Select(col => mdata.StringColumns[col]))
{
for (int i = 0; i < txt.Length; i++)
{
string s = txt[i];
foreach (string s1 in word)
{
if (s.ToLower().Contains(s1))
{
indicator[i] = true;
break;
}
}
}
}
string[][] newCol = new string[indicator.Length][];
for (int i = 0; i < newCol.Length; i++)
{
bool yes = inverse ? !indicator[i] : indicator[i];
newCol[i] = yes ? new[] { "+" } : new string[0];
}
mdata.AddCategoryColumn(colName, "", newCol);
}
public void ProcessData(IMatrixData mdata, Parameters param, ref IMatrixData[] supplTables,
ref IDocumentData[] documents, ProcessInfo processInfo)
{
ParameterWithSubParams<int> p = param.GetParamWithSubParams<int>("Column");
int colInd = p.Value;
if (colInd < 0){
processInfo.ErrString = "No categorical columns available.";
return;
}
Parameter<int[]> mcp = p.GetSubParameters().GetParam<int[]>("Values");
int[] inds = mcp.Value;
if (inds.Length == 0){
processInfo.ErrString = "Please select at least one term for filtering.";
return;
}
string[] values = new string[inds.Length];
string[] v = mdata.GetCategoryColumnValuesAt(colInd);
for (int i = 0; i < values.Length; i++){
values[i] = v[inds[i]];
}
HashSet<string> value = new HashSet<string>(values);
bool remove = param.GetParam<int>("Mode").Value == 0;
List<int> valids = new List<int>();
for (int i = 0; i < mdata.RowCount; i++){
bool valid = true;
foreach (string w in mdata.GetCategoryColumnEntryAt(colInd, i)){
if (value.Contains(w)){
valid = false;
break;
}
}
if ((valid && remove) || (!valid && !remove)){
valids.Add(i);
}
}
PerseusPluginUtils.FilterRows(mdata, param, valids.ToArray());
}
public static int[] GetIndicesOf(IMatrixData data, string categoryName, HashSet<string> values)
{
int index = GetIndexOf(data, categoryName);
List<int> result = new List<int>();
for (int i = 0; i < data.RowCount; i++){
string[] s = data.GetCategoryColumnEntryAt(index, i);
foreach (string s1 in s){
if (values.Contains(s1)){
result.Add(i);
break;
}
}
}
return result.ToArray();
}