// Does RowNormalize, but doesn't kernel normalize
public Matrix <string, string, double> ToSubKernel(Matrix <string, string, double> unnormalizedAndUnfilteredInput, int cidInBatchCount, int batchCount, int i, int j, List <EScienceLearn.RowFilters.RowFilter> RowFilterList)
{
//!!!could cache the twoMatrices, so if one or both is the same, wouldn't reload.
var unfilteredInput = RowNormalizer.Normalize(unnormalizedAndUnfilteredInput);
var input = EScienceLearn.RowFilters.RowFilter.ApplyList(RowFilterList, unfilteredInput, null);
var cidListList = SpecialFunctions.DivideListIntoEqualChunksFromChunkSize <string>(input.ColKeys, cidInBatchCount);
Helper.CheckCondition(cidListList.Count == batchCount, "Expect the cidListList.Count == batchCount");
Console.WriteLine("cids divided into {0} batches of about {1}", cidListList.Count, cidInBatchCount);
Helper.CheckCondition(cidListList.Sum(l => l.Count) == input.ColCount, "real assert");
Console.WriteLine("Loading batch {0}, size {1}x{2}", i, cidListList[i].Count, input.RowCount);
var matrixI = input.SelectColsView(cidListList[i]).ToShoMatrix(verbose: true);
Console.WriteLine("Loading batch {0}, size {1}x{2}", j, cidListList[j].Count, input.RowCount);
if (i == j)
{
ShoMatrix kii = JustKernel(matrixI);
return(kii);
}
else
{
var matrixJ = input.SelectColsView(cidListList[j]).ToShoMatrix(verbose: true);
ShoMatrix kij = JustKernel(matrixI, matrixJ);
return(kij);
}
}
public override void NormalizeInPlace(ref ShoMatrix shoMatrix)
{
DoubleArray K = shoMatrix.DoubleArray;
Cholesky tmp = K.TryCholesky();
Helper.CheckCondition(null != tmp, "Expect matrix to be positive definite");
}
private Matrix <string, string, double> ToKernel(Matrix <string, string, double> matrixIn)
{
ShoMatrix shoMatrix = matrixIn.ToShoMatrix(); //Will always make a copy
Matrix <string, string, double> kernel = ToKernel(ref shoMatrix);
return(kernel);
}
public void NormalizeInPlace(ref ShoMatrix matrix)
{
var matrix2 = matrix;
DoubleArray doubleArray = matrix.DoubleArray;
NormalizeInPlace(ref doubleArray, rowIndex => matrix2.RowKeys[rowIndex]);
matrix.DoubleArray = doubleArray;
}
private Matrix <string, string, double> ToKernel(ref ShoMatrix shoMatrix)
{
RowNormalizer.NormalizeInPlace(ref shoMatrix);
Matrix <string, string, double> kernel = JustKernel(shoMatrix);
KernelNormalizeInPlace(ref kernel);
return(kernel);
}
private ShoMatrix JustKernel(ShoMatrix normalizedShoMatrix)
{
Helper.CheckCondition(double.IsNaN(normalizedShoMatrix.MissingValue), "Expect shoMatrix's missing value to be NaN");
DoubleArray normalizedArray = normalizedShoMatrix.DoubleArray;
DoubleArray doubleArrayKernel = JustKernel(normalizedArray);
ShoMatrix kernel = new ShoMatrix(doubleArrayKernel, normalizedShoMatrix.ColKeys, normalizedShoMatrix.ColKeys, double.NaN);
return(kernel);
}
public static List <string> IntersectOnColKeys(ref ShoMatrix m1, ref Matrix <string, string, double> m2)
{
IList <string> cidsPhen = m2.ColKeys;
IList <string> cidsSnp = m1.ColKeys;
List <string> cids = cidsPhen.Intersect(cidsSnp).ToList();
m1 = m1.SelectColsView(cids).AsShoMatrix();
m2 = m2.SelectColsView(cids).AsShoMatrix();
return(cids);
}
private ShoMatrix JustKernel(ShoMatrix normalizedShoMatrixA, ShoMatrix normalizedShoMatrixB)
{
Helper.CheckCondition(double.IsNaN(normalizedShoMatrixA.MissingValue) && double.IsNaN(normalizedShoMatrixA.MissingValue), "Expect the input matrices' missing value to be NaN");
DoubleArray normalizedArrayA = normalizedShoMatrixA.DoubleArray;
DoubleArray normalizedArrayB = normalizedShoMatrixB.DoubleArray;
DoubleArray doubleArrayKernel = JustKernel(normalizedArrayA, normalizedArrayB);
ShoMatrix kernel = new ShoMatrix(doubleArrayKernel, normalizedShoMatrixA.ColKeys, normalizedShoMatrixB.ColKeys, double.NaN);
return(kernel);
}
public void NormalizeInPlace(ref ShoMatrix matrix, ParallelOptions parallelOptions)
{
var matrix2 = matrix;
using (ParallelOptionsScope.Create(parallelOptions))
{
NormalizeInPlace(ref matrix2);
}
matrix = matrix2;
}
public static List <string> IntersectOnRowKeysShoMatrix(ref ShoMatrix m1, ref ShoMatrix m2)
{
IList <string> var2 = m2.RowKeys;
IList <string> var1 = m1.RowKeys;
List <string> var = var2.Intersect(var1).ToList();
m1 = m1.SelectRowsView(var).AsShoMatrix();
m2 = m2.SelectRowsView(var).AsShoMatrix();
return(var);
}
public Matrix <string, string, double> ToKernel(Matrix <string, string, double> unnormalizedInput, int?cidInBatchCountOrNull = null)
{
if (null == cidInBatchCountOrNull)
{
return(ToKernel(unnormalizedInput));
}
var input = RowNormalizer.Normalize(unnormalizedInput);
var cidListList = SpecialFunctions.DivideListIntoEqualChunksFromChunkSize <string>(input.ColKeys, cidInBatchCountOrNull.Value);
Console.WriteLine("cids divided into {0} batches of about {1}", cidListList.Count, cidInBatchCountOrNull);
Helper.CheckCondition(cidListList.Sum(l => l.Count) == input.ColCount, "real assert");
var counterWithMessages = new CounterWithMessages("kernel combintations ", 1, (cidListList.Count * cidListList.Count + cidListList.Count) / 2);
var kernelPieces2D = new Matrix <string, string, double> [cidListList.Count, cidListList.Count];
for (int i = 0; i < cidListList.Count; ++i)
{
Console.WriteLine("Loading batch {0}, size {1}x{2}", i, cidListList[i].Count, unnormalizedInput.RowCount);
var matrixI = input.SelectColsView(cidListList[i]).ToShoMatrix(verbose: true);
Parallel.For(i, cidListList.Count, ParallelOptionsScope.Current, j =>
{
Console.WriteLine("Loading batch {0}, size {1}x{2}", j, cidListList[j].Count, unnormalizedInput.RowCount);
if (i == j)
{
ShoMatrix kii = JustKernel(matrixI);
kernelPieces2D[i, i] = kii;
}
else
{
var matrixJ = input.SelectColsView(cidListList[j]).ToShoMatrix(verbose: true);
ShoMatrix kij = JustKernel(matrixI, matrixJ);
kernelPieces2D[i, j] = kij;
kernelPieces2D[j, i] = kij.TransposeView().ToShoMatrix();
}
counterWithMessages.Increment();
});
}
counterWithMessages.Finished();
var output = MatrixExtensions.MergeRowsAndColsView(kernelPieces2D);
Helper.CheckCondition(output.RowKeys.SequenceEqual(output.ColKeys) && output.ColKeys.SequenceEqual(unnormalizedInput.ColKeys), "Assert: MergeRows isn't working as expected");
KernelNormalizeInPlace(ref output);
return(output);
}
static public ShoMatrix Standardize <TMatrix>(this TMatrix matrix, int maxMatrixVal, ParallelOptions parallelOptions) where TMatrix : Matrix <string, string, double>
{
if (matrix.RowCount > 0)
{
ShoMatrix snpMatrixNew = EigenstratMain.StandardizeGToCreateX <ShoMatrix>(maxMatrixVal, matrix, ShoMatrix.CreateDefaultInstance, parallelOptions);
return(snpMatrixNew);
}
else
{
return(new ShoMatrix(matrix));
}
}
public override void NormalizeInPlace(ref ShoMatrix shoMatrix)
{
if (Fudge == 0)
{
return;
}
DoubleArray doubleArray = shoMatrix.DoubleArray;
Helper.CheckCondition(doubleArray.size0 == doubleArray.size1, "Expect #rows == #cols");
for (int i = 0; i < doubleArray.size0; ++i)
{
doubleArray[i, i] += Fudge;
}
}
public override void NormalizeInPlace(ref ShoMatrix shoMatrix)
{
DoubleArray x = shoMatrix.DoubleArray;
var invSqrtDiagonal = x.Diagonal.Select(v => 1.0 / Math.Sqrt(v)).ToArray();
Parallel.For(0, x.size0, ParallelOptionsScope.Current, r =>
{
for (int c = 0; c < x.size1; c++)
{
x[r, c] *= invSqrtDiagonal[r] * invSqrtDiagonal[c];
}
});
}
/// <summary>
/// Add a bunch of 1's in the last row, and call this row key "bias"--useful for regression
/// </summary>
/// <returns></returns>
public static Matrix <string, string, double> AddBiasRow(this Matrix <string, string, double> x)
{
DoubleArray biasArray = ShoUtils.DoubleArrayOnes(1, x.ColCount);//new DoubleArray(1,x.ColCount);
ShoMatrix biasMatrix = new ShoMatrix(biasArray, new List <string> {
"bias"
}, x.ColKeys, x.MissingValue);
return(x.MergeRowsView <string, string, double>(true, biasMatrix));
//DoubleArray newD = ShoUtils.AddBiasToInputData(this.DoubleArray.T, this.ColCount).T;
//List<string> newRowKeys = this.RowKeys.ToList();
//newRowKeys.Add("bias");
//ShoMatrix x = new ShoMatrix(newD, newRowKeys, this.ColKeys, this.MissingValue);
//return x;
}
//for each row, average any columns that have the same groupId if numRepeat=-1
public static ShoMatrix AverageColSubsetsInSameGroup(this ShoMatrix M, Matrix <string, string, string> groupIds, out ShoMatrix numInstancesPerGroup, ParallelOptions parallelOptions, int repeatNum)
{
M.CheckEqualColKeys(groupIds.ColKeys.ToList());
Helper.CheckCondition(groupIds.RowCount == 1, "group ids should have only one row");
List <string> uniqueGroupIds = groupIds.Unique().ToList();
int G = uniqueGroupIds.Count;
DoubleArray averagedResults = new DoubleArray(M.RowCount, G);
DoubleArray numInstancesPerGroupArray = ShoUtils.DoubleArrayZeros(1, G);
BoolArray haveProcessedGroupG = new BoolArray(1, G);
for (int g = 0; g < G; g++)
{
string gId = uniqueGroupIds[g];
List <int> theseCols = new List <int>();
for (int n = 0; n < groupIds.ColCount; n++)
{
if (groupIds[0, n] == uniqueGroupIds[g])
{
if (repeatNum < 0 || !haveProcessedGroupG[g])//averaging, or else it is the right repeat
{
theseCols.Add(n);
haveProcessedGroupG[g] = true;
}
}
}
DoubleArray avgValues = M.DoubleArray.GetCols(theseCols).Mean(DimOp.OverCol);
averagedResults.SetCol(g, avgValues);
numInstancesPerGroupArray[0, g] = theseCols.Count();
}
numInstancesPerGroup = new ShoMatrix(numInstancesPerGroupArray, new List <string>()
{
"count"
}, uniqueGroupIds, double.NaN);
return(new ShoMatrix(averagedResults, M.RowKeys, uniqueGroupIds, double.NaN));
}
public static ShoMatrix AverageColSubsetsInSameGroup(this ShoMatrix M, Matrix <string, string, string> groupIds, out ShoMatrix numInstancesPerGroup, ParallelOptions parallelOptions)
{
return(AverageColSubsetsInSameGroup(M, groupIds, out numInstancesPerGroup, parallelOptions, -1));
}
public override void NormalizeInPlace(ref ShoMatrix shoMatrix)
{
DoubleArray x = shoMatrix.DoubleArray;
Helper.CheckCondition(x.IsSymmetric(), "Expect matrix to be symmetric");
}
abstract public void NormalizeInPlace(ref ShoMatrix shoMatrix);
public override void NormalizeInPlace(ref ShoMatrix shoMatrix)
{
Bio.Util.FileUtils.CreateDirectoryForFileIfNeeded(OutputFileName);
shoMatrix.WriteDense(OutputFileName);
}
static public ShoMatrix Standardize(this ShoMatrix shoMatrix, int maxMatrixVal, ParallelOptions parallelOptions, bool onlyMeanCenter)
{
ShoMatrix snpMatrixNew = EigenstratMain.StandardizeGToCreateX <ShoMatrix>(maxMatrixVal, shoMatrix, ShoMatrix.CreateDefaultInstance, parallelOptions, onlyMeanCenter);
return(snpMatrixNew);
}
public static List <string> IntersectOnColKeys(ref Matrix <string, string, double> m1, ref ShoMatrix m2)
{
return(IntersectOnColKeys(ref m2, ref m1));
}
public static void IntersectOnColAndRowKeysShoMatrix(ref ShoMatrix m1, ref ShoMatrix m2)
{
IntersectOnColKeysShoMatrix(ref m1, ref m2);
IntersectOnRowKeysShoMatrix(ref m1, ref m2);
}
