// Does RowNormalize, but doesn't kernel normalize public Matrix <string, string, double> ToSubKernel(Matrix <string, string, double> unnormalizedAndUnfilteredInput, int cidInBatchCount, int batchCount, int i, int j, List <EScienceLearn.RowFilters.RowFilter> RowFilterList) { //!!!could cache the twoMatrices, so if one or both is the same, wouldn't reload. var unfilteredInput = RowNormalizer.Normalize(unnormalizedAndUnfilteredInput); var input = EScienceLearn.RowFilters.RowFilter.ApplyList(RowFilterList, unfilteredInput, null); var cidListList = SpecialFunctions.DivideListIntoEqualChunksFromChunkSize <string>(input.ColKeys, cidInBatchCount); Helper.CheckCondition(cidListList.Count == batchCount, "Expect the cidListList.Count == batchCount"); Console.WriteLine("cids divided into {0} batches of about {1}", cidListList.Count, cidInBatchCount); Helper.CheckCondition(cidListList.Sum(l => l.Count) == input.ColCount, "real assert"); Console.WriteLine("Loading batch {0}, size {1}x{2}", i, cidListList[i].Count, input.RowCount); var matrixI = input.SelectColsView(cidListList[i]).ToShoMatrix(verbose: true); Console.WriteLine("Loading batch {0}, size {1}x{2}", j, cidListList[j].Count, input.RowCount); if (i == j) { ShoMatrix kii = JustKernel(matrixI); return(kii); } else { var matrixJ = input.SelectColsView(cidListList[j]).ToShoMatrix(verbose: true); ShoMatrix kij = JustKernel(matrixI, matrixJ); return(kij); } }
public override void NormalizeInPlace(ref ShoMatrix shoMatrix) { DoubleArray K = shoMatrix.DoubleArray; Cholesky tmp = K.TryCholesky(); Helper.CheckCondition(null != tmp, "Expect matrix to be positive definite"); }
private Matrix <string, string, double> ToKernel(Matrix <string, string, double> matrixIn) { ShoMatrix shoMatrix = matrixIn.ToShoMatrix(); //Will always make a copy Matrix <string, string, double> kernel = ToKernel(ref shoMatrix); return(kernel); }
public void NormalizeInPlace(ref ShoMatrix matrix) { var matrix2 = matrix; DoubleArray doubleArray = matrix.DoubleArray; NormalizeInPlace(ref doubleArray, rowIndex => matrix2.RowKeys[rowIndex]); matrix.DoubleArray = doubleArray; }
private Matrix <string, string, double> ToKernel(ref ShoMatrix shoMatrix) { RowNormalizer.NormalizeInPlace(ref shoMatrix); Matrix <string, string, double> kernel = JustKernel(shoMatrix); KernelNormalizeInPlace(ref kernel); return(kernel); }
private ShoMatrix JustKernel(ShoMatrix normalizedShoMatrix) { Helper.CheckCondition(double.IsNaN(normalizedShoMatrix.MissingValue), "Expect shoMatrix's missing value to be NaN"); DoubleArray normalizedArray = normalizedShoMatrix.DoubleArray; DoubleArray doubleArrayKernel = JustKernel(normalizedArray); ShoMatrix kernel = new ShoMatrix(doubleArrayKernel, normalizedShoMatrix.ColKeys, normalizedShoMatrix.ColKeys, double.NaN); return(kernel); }
public static List <string> IntersectOnColKeys(ref ShoMatrix m1, ref Matrix <string, string, double> m2) { IList <string> cidsPhen = m2.ColKeys; IList <string> cidsSnp = m1.ColKeys; List <string> cids = cidsPhen.Intersect(cidsSnp).ToList(); m1 = m1.SelectColsView(cids).AsShoMatrix(); m2 = m2.SelectColsView(cids).AsShoMatrix(); return(cids); }
private ShoMatrix JustKernel(ShoMatrix normalizedShoMatrixA, ShoMatrix normalizedShoMatrixB) { Helper.CheckCondition(double.IsNaN(normalizedShoMatrixA.MissingValue) && double.IsNaN(normalizedShoMatrixA.MissingValue), "Expect the input matrices' missing value to be NaN"); DoubleArray normalizedArrayA = normalizedShoMatrixA.DoubleArray; DoubleArray normalizedArrayB = normalizedShoMatrixB.DoubleArray; DoubleArray doubleArrayKernel = JustKernel(normalizedArrayA, normalizedArrayB); ShoMatrix kernel = new ShoMatrix(doubleArrayKernel, normalizedShoMatrixA.ColKeys, normalizedShoMatrixB.ColKeys, double.NaN); return(kernel); }
public void NormalizeInPlace(ref ShoMatrix matrix, ParallelOptions parallelOptions) { var matrix2 = matrix; using (ParallelOptionsScope.Create(parallelOptions)) { NormalizeInPlace(ref matrix2); } matrix = matrix2; }
public static List <string> IntersectOnRowKeysShoMatrix(ref ShoMatrix m1, ref ShoMatrix m2) { IList <string> var2 = m2.RowKeys; IList <string> var1 = m1.RowKeys; List <string> var = var2.Intersect(var1).ToList(); m1 = m1.SelectRowsView(var).AsShoMatrix(); m2 = m2.SelectRowsView(var).AsShoMatrix(); return(var); }
public Matrix <string, string, double> ToKernel(Matrix <string, string, double> unnormalizedInput, int?cidInBatchCountOrNull = null) { if (null == cidInBatchCountOrNull) { return(ToKernel(unnormalizedInput)); } var input = RowNormalizer.Normalize(unnormalizedInput); var cidListList = SpecialFunctions.DivideListIntoEqualChunksFromChunkSize <string>(input.ColKeys, cidInBatchCountOrNull.Value); Console.WriteLine("cids divided into {0} batches of about {1}", cidListList.Count, cidInBatchCountOrNull); Helper.CheckCondition(cidListList.Sum(l => l.Count) == input.ColCount, "real assert"); var counterWithMessages = new CounterWithMessages("kernel combintations ", 1, (cidListList.Count * cidListList.Count + cidListList.Count) / 2); var kernelPieces2D = new Matrix <string, string, double> [cidListList.Count, cidListList.Count]; for (int i = 0; i < cidListList.Count; ++i) { Console.WriteLine("Loading batch {0}, size {1}x{2}", i, cidListList[i].Count, unnormalizedInput.RowCount); var matrixI = input.SelectColsView(cidListList[i]).ToShoMatrix(verbose: true); Parallel.For(i, cidListList.Count, ParallelOptionsScope.Current, j => { Console.WriteLine("Loading batch {0}, size {1}x{2}", j, cidListList[j].Count, unnormalizedInput.RowCount); if (i == j) { ShoMatrix kii = JustKernel(matrixI); kernelPieces2D[i, i] = kii; } else { var matrixJ = input.SelectColsView(cidListList[j]).ToShoMatrix(verbose: true); ShoMatrix kij = JustKernel(matrixI, matrixJ); kernelPieces2D[i, j] = kij; kernelPieces2D[j, i] = kij.TransposeView().ToShoMatrix(); } counterWithMessages.Increment(); }); } counterWithMessages.Finished(); var output = MatrixExtensions.MergeRowsAndColsView(kernelPieces2D); Helper.CheckCondition(output.RowKeys.SequenceEqual(output.ColKeys) && output.ColKeys.SequenceEqual(unnormalizedInput.ColKeys), "Assert: MergeRows isn't working as expected"); KernelNormalizeInPlace(ref output); return(output); }
static public ShoMatrix Standardize <TMatrix>(this TMatrix matrix, int maxMatrixVal, ParallelOptions parallelOptions) where TMatrix : Matrix <string, string, double> { if (matrix.RowCount > 0) { ShoMatrix snpMatrixNew = EigenstratMain.StandardizeGToCreateX <ShoMatrix>(maxMatrixVal, matrix, ShoMatrix.CreateDefaultInstance, parallelOptions); return(snpMatrixNew); } else { return(new ShoMatrix(matrix)); } }
public override void NormalizeInPlace(ref ShoMatrix shoMatrix) { if (Fudge == 0) { return; } DoubleArray doubleArray = shoMatrix.DoubleArray; Helper.CheckCondition(doubleArray.size0 == doubleArray.size1, "Expect #rows == #cols"); for (int i = 0; i < doubleArray.size0; ++i) { doubleArray[i, i] += Fudge; } }
public override void NormalizeInPlace(ref ShoMatrix shoMatrix) { DoubleArray x = shoMatrix.DoubleArray; var invSqrtDiagonal = x.Diagonal.Select(v => 1.0 / Math.Sqrt(v)).ToArray(); Parallel.For(0, x.size0, ParallelOptionsScope.Current, r => { for (int c = 0; c < x.size1; c++) { x[r, c] *= invSqrtDiagonal[r] * invSqrtDiagonal[c]; } }); }
/// <summary> /// Add a bunch of 1's in the last row, and call this row key "bias"--useful for regression /// </summary> /// <returns></returns> public static Matrix <string, string, double> AddBiasRow(this Matrix <string, string, double> x) { DoubleArray biasArray = ShoUtils.DoubleArrayOnes(1, x.ColCount);//new DoubleArray(1,x.ColCount); ShoMatrix biasMatrix = new ShoMatrix(biasArray, new List <string> { "bias" }, x.ColKeys, x.MissingValue); return(x.MergeRowsView <string, string, double>(true, biasMatrix)); //DoubleArray newD = ShoUtils.AddBiasToInputData(this.DoubleArray.T, this.ColCount).T; //List<string> newRowKeys = this.RowKeys.ToList(); //newRowKeys.Add("bias"); //ShoMatrix x = new ShoMatrix(newD, newRowKeys, this.ColKeys, this.MissingValue); //return x; }
//for each row, average any columns that have the same groupId if numRepeat=-1 public static ShoMatrix AverageColSubsetsInSameGroup(this ShoMatrix M, Matrix <string, string, string> groupIds, out ShoMatrix numInstancesPerGroup, ParallelOptions parallelOptions, int repeatNum) { M.CheckEqualColKeys(groupIds.ColKeys.ToList()); Helper.CheckCondition(groupIds.RowCount == 1, "group ids should have only one row"); List <string> uniqueGroupIds = groupIds.Unique().ToList(); int G = uniqueGroupIds.Count; DoubleArray averagedResults = new DoubleArray(M.RowCount, G); DoubleArray numInstancesPerGroupArray = ShoUtils.DoubleArrayZeros(1, G); BoolArray haveProcessedGroupG = new BoolArray(1, G); for (int g = 0; g < G; g++) { string gId = uniqueGroupIds[g]; List <int> theseCols = new List <int>(); for (int n = 0; n < groupIds.ColCount; n++) { if (groupIds[0, n] == uniqueGroupIds[g]) { if (repeatNum < 0 || !haveProcessedGroupG[g])//averaging, or else it is the right repeat { theseCols.Add(n); haveProcessedGroupG[g] = true; } } } DoubleArray avgValues = M.DoubleArray.GetCols(theseCols).Mean(DimOp.OverCol); averagedResults.SetCol(g, avgValues); numInstancesPerGroupArray[0, g] = theseCols.Count(); } numInstancesPerGroup = new ShoMatrix(numInstancesPerGroupArray, new List <string>() { "count" }, uniqueGroupIds, double.NaN); return(new ShoMatrix(averagedResults, M.RowKeys, uniqueGroupIds, double.NaN)); }
public static ShoMatrix AverageColSubsetsInSameGroup(this ShoMatrix M, Matrix <string, string, string> groupIds, out ShoMatrix numInstancesPerGroup, ParallelOptions parallelOptions) { return(AverageColSubsetsInSameGroup(M, groupIds, out numInstancesPerGroup, parallelOptions, -1)); }
public override void NormalizeInPlace(ref ShoMatrix shoMatrix) { DoubleArray x = shoMatrix.DoubleArray; Helper.CheckCondition(x.IsSymmetric(), "Expect matrix to be symmetric"); }
abstract public void NormalizeInPlace(ref ShoMatrix shoMatrix);
public override void NormalizeInPlace(ref ShoMatrix shoMatrix) { Bio.Util.FileUtils.CreateDirectoryForFileIfNeeded(OutputFileName); shoMatrix.WriteDense(OutputFileName); }
static public ShoMatrix Standardize(this ShoMatrix shoMatrix, int maxMatrixVal, ParallelOptions parallelOptions, bool onlyMeanCenter) { ShoMatrix snpMatrixNew = EigenstratMain.StandardizeGToCreateX <ShoMatrix>(maxMatrixVal, shoMatrix, ShoMatrix.CreateDefaultInstance, parallelOptions, onlyMeanCenter); return(snpMatrixNew); }
public static List <string> IntersectOnColKeys(ref Matrix <string, string, double> m1, ref ShoMatrix m2) { return(IntersectOnColKeys(ref m2, ref m1)); }
public static void IntersectOnColAndRowKeysShoMatrix(ref ShoMatrix m1, ref ShoMatrix m2) { IntersectOnColKeysShoMatrix(ref m1, ref m2); IntersectOnRowKeysShoMatrix(ref m1, ref m2); }