public static void GetSpectralResiduals( IROMatrix matrixX, IROMatrix xLoads, IROMatrix yLoads, IROMatrix xScores, IROVector crossProduct, int numberOfFactors, IMatrix spectralResiduals) { int numX = xLoads.Columns; int numY = yLoads.Columns; int numM = yLoads.Rows; MatrixMath.BEMatrix reconstructedSpectra = new MatrixMath.BEMatrix(matrixX.Rows, matrixX.Columns); MatrixMath.ZeroMatrix(reconstructedSpectra); for (int nf = 0; nf < numberOfFactors; nf++) { double scale = crossProduct[nf]; for (int m = 0; m < numM; m++) { for (int k = 0; k < numX; k++) { reconstructedSpectra[m, k] += scale * xScores[m, nf] * xLoads[nf, k]; } } } for (int m = 0; m < numM; m++) { spectralResiduals[m, 0] = MatrixMath.SumOfSquaredDifferences( MatrixMath.ToROSubMatrix(matrixX, m, 0, 1, matrixX.Columns), MatrixMath.ToROSubMatrix(reconstructedSpectra, m, 0, 1, matrixX.Columns)); } }
/// <summary> /// Fits a data set linear to a given function base. /// </summary> /// <param name="xarr">The array of x values of the data set.</param> /// <param name="yarr">The array of y values of the data set.</param> /// <param name="stddev">The array of y standard deviations of the data set.</param> /// <param name="numberOfData">The number of data points (may be smaller than the array sizes of the data arrays).</param> /// <param name="numberOfParameter">The number of parameters to fit == size of the function base.</param> /// <param name="evaluateFunctionBase">The function base used to fit.</param> /// <param name="threshold">A treshold value (usually 1E-5) used to chop the unimportant singular values away.</param> public LinearFitBySvd( double[] xarr, double[] yarr, double[] stddev, int numberOfData, int numberOfParameter, FunctionBaseEvaluator evaluateFunctionBase, double threshold) { IMatrix u = new MatrixMath.BEMatrix(numberOfData, numberOfParameter); double[] functionBase = new double[numberOfParameter]; // Fill the function base matrix (rows: numberOfData, columns: numberOfParameter) // and scale also y for (int i = 0; i < numberOfData; i++) { evaluateFunctionBase(xarr[i], functionBase); for (int j = 0; j < numberOfParameter; j++) { u[i, j] = functionBase[j]; } } Calculate( u, yarr, stddev, numberOfData, numberOfParameter, threshold); }
/// <summary> /// Creates an analyis from preprocessed spectra and preprocessed concentrations. /// </summary> /// <param name="matrixX">The spectral matrix (each spectrum is a row in the matrix). They must at least be centered.</param> /// <param name="matrixY">The matrix of concentrations (each experiment is a row in the matrix). They must at least be centered.</param> /// <param name="maxFactors">Maximum number of factors for analysis.</param> /// <returns>A regression object, which holds all the loads and weights neccessary for further calculations.</returns> protected override void AnalyzeFromPreprocessedWithoutReset(IROMatrix matrixX, IROMatrix matrixY, int maxFactors) { int numberOfFactors = _calib.NumberOfFactors = Math.Min(matrixX.Columns, maxFactors); IMatrix helperY = new MatrixMath.BEMatrix(matrixY.Rows, 1); _PRESS = null; for (int i = 0; i < matrixY.Columns; i++) { MatrixMath.Submatrix(matrixY, helperY, 0, i); PLS2Regression r = PLS2Regression.CreateFromPreprocessed(matrixX, helperY, maxFactors); IPLS2CalibrationModel cal = r.CalibrationModel; _calib.NumberOfFactors = Math.Min(_calib.NumberOfFactors, cal.NumberOfFactors); _calib.XLoads[i] = cal.XLoads; _calib.YLoads[i] = cal.YLoads; _calib.XWeights[i] = cal.XWeights; _calib.CrossProduct[i] = cal.CrossProduct; if (_PRESS == null) { _PRESS = VectorMath.CreateExtensibleVector(r.PRESS.Length); } VectorMath.Add(_PRESS, r.PRESS, _PRESS); } }
public static void GetPredictionScoreMatrix( IROMatrix xLoads, IROMatrix yLoads, IROMatrix xScores, IROVector crossProduct, int numberOfFactors, IMatrix predictionScores) { int numX = xLoads.Columns; int numY = yLoads.Columns; int numM = yLoads.Rows; MatrixMath.BEMatrix UtY = new MatrixMath.BEMatrix(xScores.Columns, yLoads.Columns); MatrixMath.MultiplyFirstTransposed(xScores, yLoads, UtY); MatrixMath.ZeroMatrix(predictionScores); for (int nf = 0; nf < numberOfFactors; nf++) { double scale = 1 / crossProduct[nf]; for (int cn = 0; cn < numY; cn++) { for (int k = 0; k < numX; k++) { predictionScores[k, cn] += scale * xLoads[nf, k] * UtY[nf, cn]; } } } }
/// <summary> /// Calculates the prediction scores (for use withthe preprocessed spectra). /// </summary> /// <param name="numFactors">Number of factors used to calculate the prediction scores.</param> /// <param name="predictionScores">Supplied matrix for holding the prediction scores.</param> protected override void InternalGetPredictionScores(int numFactors, IMatrix predictionScores) { IMatrix pred = new MatrixMath.BEMatrix(predictionScores.Rows, 1); for (int i = 0; i < _calib.NumberOfY; i++) { PLS2Regression.GetPredictionScoreMatrix(_calib.XLoads[i], _calib.YLoads[i], _calib.XWeights[i], _calib.CrossProduct[i], numFactors, pred); MatrixMath.SetColumn(pred, predictionScores, i); } }
public CrossValidationResult(int numberOfPoints, int numberOfY, int numberOfFactors, bool multipleSpectralResiduals) { _predictedY = new IMatrix[numberOfFactors + 1]; _spectralResidual = new IMatrix[numberOfFactors + 1]; _crossPRESS = VectorMath.CreateExtensibleVector(numberOfFactors + 1); for (int i = 0; i <= numberOfFactors; i++) { _predictedY[i] = new MatrixMath.BEMatrix(numberOfPoints, numberOfY); _spectralResidual[i] = new MatrixMath.BEMatrix(numberOfPoints, multipleSpectralResiduals ? numberOfY : 1); } }
/// <summary> /// Creates an analyis from preprocessed spectra and preprocessed concentrations. /// </summary> /// <param name="matrixX">The spectral matrix (each spectrum is a row in the matrix). They must at least be centered.</param> /// <param name="matrixY">The matrix of concentrations (each experiment is a row in the matrix). They must at least be centered.</param> /// <param name="maxFactors">Maximum number of factors for analysis.</param> /// <returns>A regression object, which holds all the loads and weights neccessary for further calculations.</returns> protected override void AnalyzeFromPreprocessedWithoutReset(IROMatrix matrixX, IROMatrix matrixY, int maxFactors) { int numberOfFactors = _calib.NumberOfFactors = Math.Min(matrixX.Columns, maxFactors); MatrixMath.BEMatrix _xLoads = new MatrixMath.BEMatrix(0, 0); MatrixMath.BEMatrix _yLoads = new MatrixMath.BEMatrix(0, 0); MatrixMath.BEMatrix _W = new MatrixMath.BEMatrix(0, 0); MatrixMath.REMatrix _V = new MatrixMath.REMatrix(0, 0); _PRESS = VectorMath.CreateExtensibleVector(0); ExecuteAnalysis(matrixX, matrixY, ref numberOfFactors, _xLoads, _yLoads, _W, _V, _PRESS); _calib.NumberOfFactors = Math.Min(_calib.NumberOfFactors, numberOfFactors); _calib.XLoads = _xLoads; _calib.YLoads = _yLoads; _calib.XWeights = _W; _calib.CrossProduct = _V; }
public static void CalculateXLeverageFromPreprocessed( IROMatrix xScores, int numberOfFactors, IMatrix leverage) { IMatrix subscores = new MatrixMath.BEMatrix(xScores.Rows, numberOfFactors); MatrixMath.Submatrix(xScores, subscores); MatrixMath.SingularValueDecomposition decompose = new MatrixMath.SingularValueDecomposition(subscores); for (int i = 0; i < xScores.Rows; i++) { leverage[i, 0] = decompose.HatDiagonal[i]; } }
/// <summary> /// Creates an analyis from preprocessed spectra and preprocessed concentrations. /// </summary> /// <param name="matrixX">The spectral matrix (each spectrum is a row in the matrix). They must at least be centered.</param> /// <param name="matrixY">The matrix of concentrations (each experiment is a row in the matrix). They must at least be centered.</param> /// <param name="maxFactors">Maximum number of factors for analysis.</param> /// <returns>A regression object, which holds all the loads and weights neccessary for further calculations.</returns> protected override void AnalyzeFromPreprocessedWithoutReset(IROMatrix matrixX, IROMatrix matrixY, int maxFactors) { int numFactors = Math.Min(matrixX.Columns, maxFactors); IROMatrix xLoads, xScores; IROVector V; ExecuteAnalysis(matrixX, matrixY, ref numFactors, out xLoads, out xScores, out V); IMatrix yLoads = new MatrixMath.BEMatrix(matrixY.Rows, matrixY.Columns); MatrixMath.Copy(matrixY, yLoads); _calib.NumberOfFactors = numFactors; _calib.XLoads = xLoads; _calib.YLoads = yLoads; _calib.XScores = xScores; _calib.CrossProduct = V; }
} // end partial-least-squares-predict public static void CalculateXLeverageFromPreprocessed( IROMatrix matrixX, IROMatrix W, // weighting matrix int numFactors, // number of factors to use for prediction IMatrix leverage, // Matrix of predicted y-values, must be same number of rows as spectra int leverageColumn ) { // get the score matrix MatrixMath.BEMatrix weights = new MatrixMath.BEMatrix(numFactors, W.Columns); MatrixMath.Submatrix(W, weights, 0, 0); MatrixMath.BEMatrix scoresMatrix = new MatrixMath.BEMatrix(matrixX.Rows, weights.Rows); MatrixMath.MultiplySecondTransposed(matrixX, weights, scoresMatrix); MatrixMath.SingularValueDecomposition decomposition = MatrixMath.GetSingularValueDecomposition(scoresMatrix); for (int i = 0; i < matrixX.Rows; i++) { leverage[i, leverageColumn] = decomposition.HatDiagonal[i]; } }
public static void ExecuteAnalysis( IROMatrix X, // matrix of spectra (a spectra is a row of this matrix) IROMatrix Y, // matrix of concentrations (a mixture is a row of this matrix) ref int numFactors, out IROMatrix xLoads, // out: the loads of the X matrix out IROMatrix xScores, // matrix of weighting values out IROVector V // vector of cross products ) { IMatrix matrixX = new MatrixMath.BEMatrix(X.Rows, X.Columns); MatrixMath.Copy(X, matrixX); MatrixMath.SingularValueDecomposition decompose = new MatrixMath.SingularValueDecomposition(matrixX); numFactors = Math.Min(numFactors, matrixX.Columns); numFactors = Math.Min(numFactors, matrixX.Rows); xLoads = JaggedArrayMath.ToTransposedROMatrix(decompose.V, Y.Rows, X.Columns); xScores = JaggedArrayMath.ToMatrix(decompose.U, Y.Rows, Y.Rows); V = VectorMath.ToROVector(decompose.Diagonal, numFactors); }
static void CalculatePRESS( IROMatrix Y, // matrix of concentrations (a mixture is a row of this matrix) IROMatrix xLoads, // out: the loads of the X matrix IROMatrix xScores, // matrix of weighting values IROVector V, // vector of cross products int maxNumberOfFactors, IVector PRESS //vector of Y PRESS values ) { IROMatrix U = xScores; MatrixMath.BEMatrix UtY = new MatrixMath.BEMatrix(Y.Rows, Y.Columns); MatrixMath.MultiplyFirstTransposed(U, Y, UtY); MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(Y.Rows, Y.Columns); MatrixMath.BEMatrix subU = new MatrixMath.BEMatrix(Y.Rows, 1); MatrixMath.BEMatrix subY = new MatrixMath.BEMatrix(Y.Rows, Y.Columns); PRESS[0] = MatrixMath.SumOfSquares(Y); int numFactors = Math.Min(maxNumberOfFactors, V.Length); // now calculate PRESS by predicting the y // using yp = U (w*(1/w)) U' y // of course w*1/w is the identity matrix, but we use only the first factors, so using a cutted identity matrix // we precalculate the last term U'y = UtY // and multiplying with one row of U in every factor step, summing up the predictedY for (int nf = 0; nf < numFactors; nf++) { for (int cn = 0; cn < Y.Columns; cn++) { for (int k = 0; k < Y.Rows; k++) { predictedY[k, cn] += U[k, nf] * UtY[nf, cn]; } } PRESS[nf + 1] = MatrixMath.SumOfSquaredDifferences(Y, predictedY); } }
public static void Predict( IROMatrix matrixX, IROMatrix xLoads, IROMatrix yLoads, IROMatrix xScores, IROVector crossProduct, int numberOfFactors, IMatrix predictedY, IMatrix spectralResiduals) { int numX = xLoads.Columns; int numY = yLoads.Columns; int numM = yLoads.Rows; MatrixMath.BEMatrix predictionScores = new MatrixMath.BEMatrix(numX, numY); GetPredictionScoreMatrix(xLoads, yLoads, xScores, crossProduct, numberOfFactors, predictionScores); MatrixMath.Multiply(matrixX, predictionScores, predictedY); if (null != spectralResiduals) { GetSpectralResiduals(matrixX, xLoads, yLoads, xScores, crossProduct, numberOfFactors, spectralResiduals); } }
public static void CalculatePRESS( IROMatrix yLoads, IROMatrix xScores, int numberOfFactors, out IROVector press) { int numMeasurements = yLoads.Rows; IExtensibleVector PRESS = VectorMath.CreateExtensibleVector(numberOfFactors + 1); MatrixMath.BEMatrix UtY = new MatrixMath.BEMatrix(yLoads.Rows, yLoads.Columns); MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(yLoads.Rows, yLoads.Columns); press = PRESS; MatrixMath.MultiplyFirstTransposed(xScores, yLoads, UtY); // now calculate PRESS by predicting the y // using yp = U (w*(1/w)) U' y // of course w*1/w is the identity matrix, but we use only the first factors, so using a cutted identity matrix // we precalculate the last term U'y = UtY // and multiplying with one row of U in every factor step, summing up the predictedY PRESS[0] = MatrixMath.SumOfSquares(yLoads); for (int nf = 0; nf < numberOfFactors; nf++) { for (int cn = 0; cn < yLoads.Columns; cn++) { for (int k = 0; k < yLoads.Rows; k++) { predictedY[k, cn] += xScores[k, nf] * UtY[nf, cn]; } } PRESS[nf + 1] = MatrixMath.SumOfSquaredDifferences(yLoads, predictedY); } }
/// <summary> /// Partial least squares (PLS) decomposition of the matrizes X and Y. /// </summary> /// <param name="_X">The X ("spectrum") matrix, centered and preprocessed.</param> /// <param name="_Y">The Y ("concentration") matrix (centered).</param> /// <param name="numFactors">Number of factors to calculate.</param> /// <param name="xLoads">Returns the matrix of eigenvectors of X. Should be initially empty.</param> /// <param name="yLoads">Returns the matrix of eigenvectors of Y. Should be initially empty. </param> /// <param name="W">Returns the matrix of weighting values. Should be initially empty.</param> /// <param name="V">Returns the vector of cross products. Should be initially empty.</param> /// <param name="PRESS">If not null, the PRESS value of each factor is stored (vertically) here. </param> public static void ExecuteAnalysis( IROMatrix _X, // matrix of spectra (a spectra is a row of this matrix) IROMatrix _Y, // matrix of concentrations (a mixture is a row of this matrix) ref int numFactors, IBottomExtensibleMatrix xLoads, // out: the loads of the X matrix IBottomExtensibleMatrix yLoads, // out: the loads of the Y matrix IBottomExtensibleMatrix W, // matrix of weighting values IRightExtensibleMatrix V, // matrix of cross products IExtensibleVector PRESS //vector of Y PRESS values ) { // used variables: // n: number of spectra (number of tests, number of experiments) // p: number of slots (frequencies, ..) in each spectrum // m: number of constitutents (number of y values in each measurement) // X : n-p matrix of spectra (each spectra is a horizontal row) // Y : n-m matrix of concentrations const int maxIterations = 1500; // max number of iterations in one factorization step const double accuracy = 1E-12; // accuracy that should be reached between subsequent calculations of the u-vector // use the mean spectrum as first row of the W matrix MatrixMath.HorizontalVector mean = new MatrixMath.HorizontalVector(_X.Columns); // MatrixMath.ColumnsToZeroMean(X,mean); //W.AppendBottom(mean); IMatrix X = new MatrixMath.BEMatrix(_X.Rows, _X.Columns); MatrixMath.Copy(_X, X); IMatrix Y = new MatrixMath.BEMatrix(_Y.Rows, _Y.Columns); MatrixMath.Copy(_Y, Y); IMatrix u_prev = null; IMatrix w = new MatrixMath.HorizontalVector(X.Columns); // horizontal vector of X (spectral) weighting IMatrix t = new MatrixMath.VerticalVector(X.Rows); // vertical vector of X scores IMatrix u = new MatrixMath.VerticalVector(X.Rows); // vertical vector of Y scores IMatrix p = new MatrixMath.HorizontalVector(X.Columns); // horizontal vector of X loads IMatrix q = new MatrixMath.HorizontalVector(Y.Columns); // horizontal vector of Y loads int maxFactors = Math.Min(X.Columns, X.Rows); numFactors = numFactors <= 0 ? maxFactors : Math.Min(numFactors, maxFactors); if (PRESS != null) { PRESS.Append(new MatrixMath.Scalar(MatrixMath.SumOfSquares(Y))); // Press value for not decomposed Y } for (int nFactor = 0; nFactor < numFactors; nFactor++) { //Console.WriteLine("Factor_{0}:",nFactor); //Console.WriteLine("X:"+X.ToString()); //Console.WriteLine("Y:"+Y.ToString()); // 1. Use as start vector for the y score the first column of the // y-matrix MatrixMath.Submatrix(Y, u); // u is now a vertical vector of concentrations of the first constituents for (int iter = 0; iter < maxIterations; iter++) { // 2. Calculate the X (spectrum) weighting vector MatrixMath.MultiplyFirstTransposed(u, X, w); // w is a horizontal vector // 3. Normalize w to unit length MatrixMath.NormalizeRows(w); // w now has unit length // 4. Calculate X (spectral) scores MatrixMath.MultiplySecondTransposed(X, w, t); // t is a vertical vector of n numbers // 5. Calculate the Y (concentration) loading vector MatrixMath.MultiplyFirstTransposed(t, Y, q); // q is a horizontal vector of m (number of constitutents) // 5.1 Normalize q to unit length MatrixMath.NormalizeRows(q); // 6. Calculate the Y (concentration) score vector u MatrixMath.MultiplySecondTransposed(Y, q, u); // u is a vertical vector of n numbers // 6.1 Compare // Compare this with the previous one if (u_prev != null && MatrixMath.IsEqual(u_prev, u, accuracy)) { break; } if (u_prev == null) { u_prev = new MatrixMath.VerticalVector(X.Rows); } MatrixMath.Copy(u, u_prev); // stores the content of u in u_prev } // for all iterations // Store the scores of X //factors.AppendRight(t); // 7. Calculate the inner scalar (cross product) double length_of_t = MatrixMath.LengthOf(t); MatrixMath.Scalar v = new MatrixMath.Scalar(0); MatrixMath.MultiplyFirstTransposed(u, t, v); if (length_of_t != 0) { v = v / MatrixMath.Square(length_of_t); } // 8. Calculate the new loads for the X (spectral) matrix MatrixMath.MultiplyFirstTransposed(t, X, p); // p is a horizontal vector of loads // Normalize p by the spectral scores if (length_of_t != 0) { MatrixMath.MultiplyScalar(p, 1 / MatrixMath.Square(length_of_t), p); } // 9. Calculate the new residua for the X (spectral) and Y (concentration) matrix //MatrixMath.MultiplyScalar(t,length_of_t*v,t); // original t times the cross product MatrixMath.SubtractProductFromSelf(t, p, X); MatrixMath.MultiplyScalar(t, v, t); // original t times the cross product MatrixMath.SubtractProductFromSelf(t, q, Y); // to calculate residual Y // Store the loads of X and Y in the output result matrix xLoads.AppendBottom(p); yLoads.AppendBottom(q); W.AppendBottom(w); V.AppendRight(v); if (PRESS != null) { double pressValue = MatrixMath.SumOfSquares(Y); PRESS.Append(new MatrixMath.Scalar(pressValue)); } // Calculate SEPcv. If SEPcv is greater than for the actual number of factors, // break since the optimal number of factors was found. If not, repeat the calculations // with the residual matrizes for the next factor. } // for all factors }
/// <summary> /// Fits a data set linear to a given x base. /// </summary> /// <param name="xbase">The matrix of x values of the data set. Dimensions: numberOfData x numberOfParameters. The matrix is changed during calculation!</param> /// <param name="yarr">The array of y values of the data set.</param> /// <param name="stddev">The array of y standard deviations of the data set.</param> /// <param name="numberOfData">The number of data points (may be smaller than the array sizes of the data arrays).</param> /// <param name="numberOfParameter">The number of parameters to fit == size of the function base.</param> /// <param name="threshold">A treshold value (usually 1E-5) used to chop the unimportant singular values away.</param> public LinearFitBySvd Calculate( IROMatrix xbase, // NumberOfData, NumberOfParameters double[] yarr, double[] stddev, int numberOfData, int numberOfParameter, double threshold) { _numberOfParameter = numberOfParameter; _numberOfFreeParameter = numberOfParameter; _numberOfData = numberOfData; _parameter = new double[numberOfParameter]; _residual = new double[numberOfData]; _predicted = new double[numberOfData]; _reducedPredictionVariance = new double[numberOfData]; double[] scaledY = new double[numberOfData]; // Calculated some useful values _yMean = Mean(yarr, 0, _numberOfData); _yCorrectedSumOfSquares = CorrectedSumOfSquares(yarr, _yMean, 0, _numberOfData); MatrixMath.BEMatrix u = new MatrixMath.BEMatrix(numberOfData, numberOfParameter); // Fill the function base matrix (rows: numberOfData, columns: numberOfParameter) // and scale also y for (int i = 0; i < numberOfData; i++) { double scale = 1 / stddev[i]; for (int j = 0; j < numberOfParameter; j++) { u[i, j] = scale * xbase[i, j]; } scaledY[i] = scale * yarr[i]; } _decomposition = MatrixMath.GetSingularValueDecomposition(u); // set singular values < thresholdLevel to zero // ChopSingularValues makes only sense if all columns of the x matrix have the same variance //decomposition.ChopSingularValues(1E-5); // recalculate the parameters with the chopped singular values _decomposition.Backsubstitution(scaledY, _parameter); _chiSquare = 0; for (int i = 0; i < numberOfData; i++) { double ypredicted = 0; for (int j = 0; j < numberOfParameter; j++) { ypredicted += _parameter[j] * xbase[i, j]; } double deviation = yarr[i] - ypredicted; _predicted[i] = ypredicted; _residual[i] = deviation; _chiSquare += deviation * deviation; } _covarianceMatrix = _decomposition.GetCovariances(); //calculate the reduced prediction variance x'(X'X)^(-1)x for (int i = 0; i < numberOfData; i++) { double total = 0; for (int j = 0; j < numberOfParameter; j++) { double sum = 0; for (int k = 0; k < numberOfParameter; k++) { sum += _covarianceMatrix[j][k] * u[i, k]; } total += u[i, j] * sum; } _reducedPredictionVariance[i] = total; } return(this); }
/// <summary> /// Multiplies selected columns to form a matrix. /// </summary> /// <param name="mainDocument"></param> /// <param name="srctable"></param> /// <param name="selectedColumns"></param> /// <returns>Null if successful, else the description of the error.</returns> /// <remarks>The user must select an even number of columns. All columns of the first half of the selection /// must have the same number of rows, and all columns of the second half of selection must also have the same /// number of rows. The first half of selected columns form a matrix of dimensions(firstrowcount,halfselected), and the second half /// of selected columns form a matrix of dimension(halfselected, secondrowcount). The resulting matrix has dimensions (firstrowcount,secondrowcount) and is /// stored in a separate worksheet.</remarks> public static string MultiplyColumnsToMatrix( Altaxo.AltaxoDocument mainDocument, Altaxo.Data.DataTable srctable, IAscendingIntegerCollection selectedColumns ) { // check that there are columns selected if (0 == selectedColumns.Count) { return("You must select at least two columns to multiply!"); } // selected columns must contain an even number of columns if (0 != selectedColumns.Count % 2) { return("You selected an odd number of columns. Please select an even number of columns to multiply!"); } // all selected columns must be numeric columns for (int i = 0; i < selectedColumns.Count; i++) { if (!(srctable[selectedColumns[i]] is Altaxo.Data.INumericColumn)) { return(string.Format("The column[{0}] (name:{1}) is not a numeric column!", selectedColumns[i], srctable[selectedColumns[i]].Name)); } } int halfselect = selectedColumns.Count / 2; // check that all columns from the first half of selected colums contain the same // number of rows int rowsfirsthalf = int.MinValue; for (int i = 0; i < halfselect; i++) { int idx = selectedColumns[i]; if (rowsfirsthalf < 0) { rowsfirsthalf = srctable[idx].Count; } else if (rowsfirsthalf != srctable[idx].Count) { return("The first half of selected columns have not all the same length!"); } } int rowssecondhalf = int.MinValue; for (int i = halfselect; i < selectedColumns.Count; i++) { int idx = selectedColumns[i]; if (rowssecondhalf < 0) { rowssecondhalf = srctable[idx].Count; } else if (rowssecondhalf != srctable[idx].Count) { return("The second half of selected columns have not all the same length!"); } } // now create the matrices to multiply from the MatrixMath.REMatrix firstMat = new MatrixMath.REMatrix(rowsfirsthalf, halfselect); for (int i = 0; i < halfselect; i++) { Altaxo.Data.INumericColumn col = (Altaxo.Data.INumericColumn)srctable[selectedColumns[i]]; for (int j = 0; j < rowsfirsthalf; j++) { firstMat[j, i] = col[j]; } } MatrixMath.BEMatrix secondMat = new MatrixMath.BEMatrix(halfselect, rowssecondhalf); for (int i = 0; i < halfselect; i++) { Altaxo.Data.INumericColumn col = (Altaxo.Data.INumericColumn)srctable[selectedColumns[i + halfselect]]; for (int j = 0; j < rowssecondhalf; j++) { secondMat[i, j] = col[j]; } } // now multiply the two matrices MatrixMath.BEMatrix resultMat = new MatrixMath.BEMatrix(rowsfirsthalf, rowssecondhalf); MatrixMath.Multiply(firstMat, secondMat, resultMat); // and store the result in a new worksheet Altaxo.Data.DataTable table = new Altaxo.Data.DataTable("ResultMatrix of " + srctable.Name); table.Suspend(); // first store the factors for (int i = 0; i < resultMat.Columns; i++) { Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn(); for (int j = 0; j < resultMat.Rows; j++) { col[j] = resultMat[j, i]; } table.DataColumns.Add(col, i.ToString()); } table.Resume(); mainDocument.DataTableCollection.Add(table); // create a new worksheet without any columns Current.ProjectService.CreateNewWorksheet(table); return(null); }
/// <summary> /// Makes a PCA (a principal component analysis) of the table or the selected columns / rows and stores the results in a newly created table. /// </summary> /// <param name="mainDocument">The main document of the application.</param> /// <param name="srctable">The table where the data come from.</param> /// <param name="selectedColumns">The selected columns.</param> /// <param name="selectedRows">The selected rows.</param> /// <param name="bHorizontalOrientedSpectrum">True if a spectrum is a single row, False if a spectrum is a single column.</param> /// <param name="maxNumberOfFactors">The maximum number of factors to calculate.</param> /// <returns></returns> public static string PrincipalComponentAnalysis( Altaxo.AltaxoDocument mainDocument, Altaxo.Data.DataTable srctable, IAscendingIntegerCollection selectedColumns, IAscendingIntegerCollection selectedRows, bool bHorizontalOrientedSpectrum, int maxNumberOfFactors ) { bool bUseSelectedColumns = (null != selectedColumns && 0 != selectedColumns.Count); int prenumcols = bUseSelectedColumns ? selectedColumns.Count : srctable.DataColumns.ColumnCount; // check for the number of numeric columns int numcols = 0; for (int i = 0; i < prenumcols; i++) { int idx = bUseSelectedColumns ? selectedColumns[i] : i; if (srctable[i] is Altaxo.Data.INumericColumn) { numcols++; } } // check the number of rows bool bUseSelectedRows = (null != selectedRows && 0 != selectedRows.Count); int numrows; if (bUseSelectedRows) { numrows = selectedRows.Count; } else { numrows = 0; for (int i = 0; i < numcols; i++) { int idx = bUseSelectedColumns ? selectedColumns[i] : i; numrows = Math.Max(numrows, srctable[idx].Count); } } // check that both dimensions are at least 2 - otherwise PCA is not possible if (numrows < 2) { return("At least two rows are neccessary to do Principal Component Analysis!"); } if (numcols < 2) { return("At least two numeric columns are neccessary to do Principal Component Analysis!"); } // Create a matrix of appropriate dimensions and fill it MatrixMath.BEMatrix matrixX; if (bHorizontalOrientedSpectrum) { matrixX = new MatrixMath.BEMatrix(numrows, numcols); int ccol = 0; // current column in the matrix for (int i = 0; i < prenumcols; i++) { int colidx = bUseSelectedColumns ? selectedColumns[i] : i; Altaxo.Data.INumericColumn col = srctable[colidx] as Altaxo.Data.INumericColumn; if (null != col) { for (int j = 0; j < numrows; j++) { int rowidx = bUseSelectedRows ? selectedRows[j] : j; matrixX[j, ccol] = col[rowidx]; } ++ccol; } } } // end if it was a horizontal oriented spectrum else // if it is a vertical oriented spectrum { matrixX = new MatrixMath.BEMatrix(numcols, numrows); int ccol = 0; // current column in the matrix for (int i = 0; i < prenumcols; i++) { int colidx = bUseSelectedColumns ? selectedColumns[i] : i; Altaxo.Data.INumericColumn col = srctable[colidx] as Altaxo.Data.INumericColumn; if (null != col) { for (int j = 0; j < numrows; j++) { int rowidx = bUseSelectedRows ? selectedRows[j] : j; matrixX[ccol, j] = col[rowidx]; } ++ccol; } } } // if it was a vertical oriented spectrum // now do PCA with the matrix MatrixMath.REMatrix factors = new MatrixMath.REMatrix(0, 0); MatrixMath.BEMatrix loads = new MatrixMath.BEMatrix(0, 0); MatrixMath.BEMatrix residualVariances = new MatrixMath.BEMatrix(0, 0); MatrixMath.HorizontalVector meanX = new MatrixMath.HorizontalVector(matrixX.Columns); // first, center the matrix MatrixMath.ColumnsToZeroMean(matrixX, meanX); MatrixMath.NIPALS_HO(matrixX, maxNumberOfFactors, 1E-9, factors, loads, residualVariances); // now we have to create a new table where to place the calculated factors and loads // we will do that in a vertical oriented manner, i.e. even if the loads are // here in horizontal vectors: in our table they are stored in (vertical) columns Altaxo.Data.DataTable table = new Altaxo.Data.DataTable("PCA of " + srctable.Name); // Fill the Table table.Suspend(); // first of all store the meanscore { double meanScore = MatrixMath.LengthOf(meanX); MatrixMath.NormalizeRows(meanX); Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn(); for (int i = 0; i < factors.Rows; i++) { col[i] = meanScore; } table.DataColumns.Add(col, "MeanFactor", Altaxo.Data.ColumnKind.V, 0); } // first store the factors for (int i = 0; i < factors.Columns; i++) { Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn(); for (int j = 0; j < factors.Rows; j++) { col[j] = factors[j, i]; } table.DataColumns.Add(col, "Factor" + i.ToString(), Altaxo.Data.ColumnKind.V, 1); } // now store the mean of the matrix { Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn(); for (int j = 0; j < meanX.Columns; j++) { col[j] = meanX[0, j]; } table.DataColumns.Add(col, "MeanLoad", Altaxo.Data.ColumnKind.V, 2); } // now store the loads - careful - they are horizontal in the matrix for (int i = 0; i < loads.Rows; i++) { Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn(); for (int j = 0; j < loads.Columns; j++) { col[j] = loads[i, j]; } table.DataColumns.Add(col, "Load" + i.ToString(), Altaxo.Data.ColumnKind.V, 3); } // now store the residual variances, they are vertical in the vector { Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn(); for (int i = 0; i < residualVariances.Rows; i++) { col[i] = residualVariances[i, 0]; } table.DataColumns.Add(col, "ResidualVariance", Altaxo.Data.ColumnKind.V, 4); } table.Resume(); mainDocument.DataTableCollection.Add(table); // create a new worksheet without any columns Current.ProjectService.CreateNewWorksheet(table); return(null); }
public DynamicParameterEstimation Calculate(IROVector x, IROVector y, int numX, int numY, int backgroundOrder) { _numX = numX; _numY = numY; _backgroundOrderPlus1 = 1 + Math.Max(-1, backgroundOrder); // where to start the calculation (index of first y point that can be used) int start = Math.Max(_numX - 1, _numY); int numberOfData = 0; if (numX > 0) { numberOfData = Math.Min(x.Length, y.Length) - start; } else { numberOfData = y.Length - start; } int numberOfParameter = _numX + _numY + _backgroundOrderPlus1; MatrixMath.BEMatrix u = new MatrixMath.BEMatrix(numberOfData, numberOfParameter); // Fill the matrix for (int i = 0; i < numberOfData; i++) { int yIdx = i + start; // x for (int j = 0; j < _numX; j++) { u[i, j] = x[yIdx - j]; } // y for (int j = 0; j < _numY; j++) { u[i, j + _numX] = y[yIdx - 1 - j]; } // polynomial background component double background = 1; for (int j = 0; j < _backgroundOrderPlus1; j++) { u[i, j + _numX + _numY] = background; background *= yIdx; } } // Fill the y double[] scaledY = new double[numberOfData]; for (int i = 0; i < numberOfData; i++) { scaledY[i] = y[i + start]; } _decomposition = MatrixMath.GetSingularValueDecomposition(u); // set singular values < thresholdLevel to zero // ChopSingularValues makes only sense if all columns of the x matrix have the same variance //decomposition.ChopSingularValues(1E-5); // recalculate the parameters with the chopped singular values _parameter = new double[numberOfParameter]; _decomposition.Backsubstitution(scaledY, _parameter); return(this); }