/// <summary>
/// Creates an analyis from preprocessed spectra and preprocessed concentrations.
/// </summary>
/// <param name="matrixX">The spectral matrix (each spectrum is a row in the matrix). They must at least be centered.</param>
/// <param name="matrixY">The matrix of concentrations (each experiment is a row in the matrix). They must at least be centered.</param>
/// <param name="maxFactors">Maximum number of factors for analysis.</param>
/// <returns>A regression object, which holds all the loads and weights neccessary for further calculations.</returns>
protected override void AnalyzeFromPreprocessedWithoutReset(IROMatrix matrixX, IROMatrix matrixY, int maxFactors)
{
int numberOfFactors = _calib.NumberOfFactors = Math.Min(matrixX.Columns, maxFactors);
IMatrix helperY = new MatrixMath.BEMatrix(matrixY.Rows,1);
_PRESS = null;
for(int i=0;i<matrixY.Columns;i++)
{
MatrixMath.Submatrix(matrixY,helperY,0,i);
PLS2Regression r = PLS2Regression.CreateFromPreprocessed(matrixX, helperY, maxFactors);
IPLS2CalibrationModel cal = r.CalibrationModel;
_calib.NumberOfFactors = Math.Min(_calib.NumberOfFactors,cal.NumberOfFactors);
_calib.XLoads[i] = cal.XLoads;
_calib.YLoads[i] = cal.YLoads;
_calib.XWeights[i] = cal.XWeights;
_calib.CrossProduct[i] = cal.CrossProduct;
if(_PRESS==null)
_PRESS = VectorMath.CreateExtensibleVector(r.PRESS.Length);
VectorMath.Add(_PRESS,r.PRESS,_PRESS);
}
}
public CrossValidationResult(int numberOfPoints, int numberOfY, int numberOfFactors, bool multipleSpectralResiduals)
{
_predictedY = new IMatrix[numberOfFactors+1];
_spectralResidual = new IMatrix[numberOfFactors+1];
_crossPRESS = VectorMath.CreateExtensibleVector(numberOfFactors+1);
for(int i=0;i<=numberOfFactors;i++)
{
_predictedY[i] = new MatrixMath.BEMatrix(numberOfPoints,numberOfY);
_spectralResidual[i] = new MatrixMath.BEMatrix(numberOfPoints,multipleSpectralResiduals ? numberOfY : 1);
}
}
/// <summary>
/// This predicts the selected columns/rows against a user choosen calibration model.
/// The orientation of spectra is given by the parameter <c>spectrumIsRow</c>.
/// </summary>
/// <param name="srctable">Table holding the specta to predict values for.</param>
/// <param name="selectedColumns">Columns selected in the source table.</param>
/// <param name="selectedRows">Rows selected in the source table.</param>
/// <param name="destTable">The table to store the prediction result.</param>
/// <param name="modelTable">The table where the calibration model is stored.</param>
/// <param name="numberOfFactors">Number of factors used to predict the values.</param>
/// <param name="spectrumIsRow">If true, the spectra is horizontally oriented, else it is vertically oriented.</param>
public virtual void PredictValues(
DataTable srctable,
IAscendingIntegerCollection selectedColumns,
IAscendingIntegerCollection selectedRows,
bool spectrumIsRow,
int numberOfFactors,
DataTable modelTable,
DataTable destTable)
{
IMultivariateCalibrationModel calibModel = GetCalibrationModel(modelTable);
// Export(modelTable, out calibModel);
var memento = GetContentAsMultivariateContentMemento(modelTable);
// Fill matrixX with spectra
Altaxo.Collections.AscendingIntegerCollection spectralIndices;
Altaxo.Collections.AscendingIntegerCollection measurementIndices;
spectralIndices = new Altaxo.Collections.AscendingIntegerCollection(selectedColumns);
measurementIndices = new Altaxo.Collections.AscendingIntegerCollection(selectedRows);
RemoveNonNumericCells(srctable, measurementIndices, spectralIndices);
// exchange selection if spectrum is column
if (!spectrumIsRow)
{
Altaxo.Collections.AscendingIntegerCollection hlp;
hlp = spectralIndices;
spectralIndices = measurementIndices;
measurementIndices = hlp;
}
// if there are more data than expected, we have to map the spectral indices
if (spectralIndices.Count > calibModel.NumberOfX)
{
double[] xofx = GetXOfSpectra(srctable, spectrumIsRow, spectralIndices, measurementIndices);
string errormsg;
AscendingIntegerCollection map = MapSpectralX(calibModel.PreprocessingModel.XOfX, VectorMath.ToROVector(xofx), out errormsg);
if (map == null)
throw new ApplicationException("Can not map spectral data: " + errormsg);
else
{
AscendingIntegerCollection newspectralindices = new AscendingIntegerCollection();
for (int i = 0; i < map.Count; i++)
newspectralindices.Add(spectralIndices[map[i]]);
spectralIndices = newspectralindices;
}
}
IMatrix matrixX = GetRawSpectra(srctable, spectrumIsRow, spectralIndices, measurementIndices);
MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(measurementIndices.Count, calibModel.NumberOfY);
CalculatePredictedY(calibModel, memento.SpectralPreprocessing, matrixX, numberOfFactors, predictedY, null);
// now save the predicted y in the destination table
Altaxo.Data.DoubleColumn labelCol = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < measurementIndices.Count; i++)
{
labelCol[i] = measurementIndices[i];
}
destTable.DataColumns.Add(labelCol, "MeasurementLabel", Altaxo.Data.ColumnKind.Label, 0);
for (int k = 0; k < predictedY.Columns; k++)
{
Altaxo.Data.DoubleColumn predictedYcol = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < measurementIndices.Count; i++)
{
predictedYcol[i] = predictedY[i, k];
}
destTable.DataColumns.Add(predictedYcol, "Predicted Y" + k.ToString(), Altaxo.Data.ColumnKind.V, 0);
}
}
/// <summary>
/// Fits a data set linear to a given function base.
/// </summary>
/// <param name="xarr">The array of x values of the data set.</param>
/// <param name="yarr">The array of y values of the data set.</param>
/// <param name="stddev">The array of y standard deviations of the data set.</param>
/// <param name="numberOfData">The number of data points (may be smaller than the array sizes of the data arrays).</param>
/// <param name="numberOfParameter">The number of parameters to fit == size of the function base.</param>
/// <param name="evaluateFunctionBase">The function base used to fit.</param>
/// <param name="threshold">A treshold value (usually 1E-5) used to chop the unimportant singular values away.</param>
public LinearFitBySvd(
double[] xarr,
double[] yarr,
double[] stddev,
int numberOfData,
int numberOfParameter,
FunctionBaseEvaluator evaluateFunctionBase,
double threshold)
{
IMatrix u = new MatrixMath.BEMatrix( numberOfData, numberOfParameter);
double[] functionBase = new double[numberOfParameter];
// Fill the function base matrix (rows: numberOfData, columns: numberOfParameter)
// and scale also y
for(int i=0;i<numberOfData;i++)
{
evaluateFunctionBase(xarr[i], functionBase);
for(int j=0;j<numberOfParameter;j++)
u[i,j] = functionBase[j];
}
Calculate(
u,
yarr,
stddev,
numberOfData,
numberOfParameter,
threshold);
}
public static void GetPredictionScoreMatrix(
IROMatrix xLoads,
IROMatrix yLoads,
IROMatrix xScores,
IROVector crossProduct,
int numberOfFactors,
IMatrix predictionScores)
{
int numX = xLoads.Columns;
int numY = yLoads.Columns;
int numM = yLoads.Rows;
MatrixMath.BEMatrix UtY = new MatrixMath.BEMatrix(xScores.Columns,yLoads.Columns);
MatrixMath.MultiplyFirstTransposed(xScores,yLoads,UtY);
MatrixMath.ZeroMatrix(predictionScores);
for(int nf=0;nf<numberOfFactors;nf++)
{
double scale = 1/crossProduct[nf];
for(int cn=0;cn<numY;cn++)
{
for(int k=0;k<numX;k++)
predictionScores[k,cn] += scale*xLoads[nf,k]*UtY[nf,cn];
}
}
}
/// <summary>
/// Partial least squares (PLS) decomposition of the matrizes X and Y.
/// </summary>
/// <param name="_X">The X ("spectrum") matrix, centered and preprocessed.</param>
/// <param name="_Y">The Y ("concentration") matrix (centered).</param>
/// <param name="numFactors">Number of factors to calculate.</param>
/// <param name="xLoads">Returns the matrix of eigenvectors of X. Should be initially empty.</param>
/// <param name="yLoads">Returns the matrix of eigenvectors of Y. Should be initially empty. </param>
/// <param name="W">Returns the matrix of weighting values. Should be initially empty.</param>
/// <param name="V">Returns the vector of cross products. Should be initially empty.</param>
/// <param name="PRESS">If not null, the PRESS value of each factor is stored (vertically) here. </param>
public static void ExecuteAnalysis(
IROMatrix _X, // matrix of spectra (a spectra is a row of this matrix)
IROMatrix _Y, // matrix of concentrations (a mixture is a row of this matrix)
ref int numFactors,
IBottomExtensibleMatrix xLoads, // out: the loads of the X matrix
IBottomExtensibleMatrix yLoads, // out: the loads of the Y matrix
IBottomExtensibleMatrix W, // matrix of weighting values
IRightExtensibleMatrix V, // matrix of cross products
IExtensibleVector PRESS //vector of Y PRESS values
)
{
// used variables:
// n: number of spectra (number of tests, number of experiments)
// p: number of slots (frequencies, ..) in each spectrum
// m: number of constitutents (number of y values in each measurement)
// X : n-p matrix of spectra (each spectra is a horizontal row)
// Y : n-m matrix of concentrations
const int maxIterations = 1500; // max number of iterations in one factorization step
const double accuracy = 1E-12; // accuracy that should be reached between subsequent calculations of the u-vector
// use the mean spectrum as first row of the W matrix
MatrixMath.HorizontalVector mean = new MatrixMath.HorizontalVector(_X.Columns);
// MatrixMath.ColumnsToZeroMean(X,mean);
//W.AppendBottom(mean);
IMatrix X = new MatrixMath.BEMatrix(_X.Rows,_X.Columns);
MatrixMath.Copy(_X,X);
IMatrix Y = new MatrixMath.BEMatrix(_Y.Rows,_Y.Columns);
MatrixMath.Copy(_Y,Y);
IMatrix u_prev = null;
IMatrix w = new MatrixMath.HorizontalVector(X.Columns); // horizontal vector of X (spectral) weighting
IMatrix t = new MatrixMath.VerticalVector(X.Rows); // vertical vector of X scores
IMatrix u = new MatrixMath.VerticalVector(X.Rows); // vertical vector of Y scores
IMatrix p = new MatrixMath.HorizontalVector(X.Columns); // horizontal vector of X loads
IMatrix q = new MatrixMath.HorizontalVector(Y.Columns); // horizontal vector of Y loads
int maxFactors = Math.Min(X.Columns,X.Rows);
numFactors = numFactors<=0 ? maxFactors : Math.Min(numFactors,maxFactors);
if(PRESS!=null)
{
PRESS.Append(new MatrixMath.Scalar(MatrixMath.SumOfSquares(Y))); // Press value for not decomposed Y
}
for(int nFactor=0; nFactor<numFactors; nFactor++)
{
//Console.WriteLine("Factor_{0}:",nFactor);
//Console.WriteLine("X:"+X.ToString());
//Console.WriteLine("Y:"+Y.ToString());
// 1. Use as start vector for the y score the first column of the
// y-matrix
MatrixMath.Submatrix(Y,u); // u is now a vertical vector of concentrations of the first constituents
for(int iter=0;iter<maxIterations;iter++)
{
// 2. Calculate the X (spectrum) weighting vector
MatrixMath.MultiplyFirstTransposed(u,X,w); // w is a horizontal vector
// 3. Normalize w to unit length
MatrixMath.NormalizeRows(w); // w now has unit length
// 4. Calculate X (spectral) scores
MatrixMath.MultiplySecondTransposed(X,w,t); // t is a vertical vector of n numbers
// 5. Calculate the Y (concentration) loading vector
MatrixMath.MultiplyFirstTransposed(t,Y,q); // q is a horizontal vector of m (number of constitutents)
// 5.1 Normalize q to unit length
MatrixMath.NormalizeRows(q);
// 6. Calculate the Y (concentration) score vector u
MatrixMath.MultiplySecondTransposed(Y,q,u); // u is a vertical vector of n numbers
// 6.1 Compare
// Compare this with the previous one
if(u_prev!=null && MatrixMath.IsEqual(u_prev,u,accuracy))
break;
if(u_prev==null)
u_prev = new MatrixMath.VerticalVector(X.Rows);
MatrixMath.Copy(u,u_prev); // stores the content of u in u_prev
} // for all iterations
// Store the scores of X
//factors.AppendRight(t);
// 7. Calculate the inner scalar (cross product)
double length_of_t = MatrixMath.LengthOf(t);
MatrixMath.Scalar v = new MatrixMath.Scalar(0);
MatrixMath.MultiplyFirstTransposed(u,t,v);
if(length_of_t!=0)
v = v/MatrixMath.Square(length_of_t);
// 8. Calculate the new loads for the X (spectral) matrix
MatrixMath.MultiplyFirstTransposed(t,X,p); // p is a horizontal vector of loads
// Normalize p by the spectral scores
if(length_of_t!=0)
MatrixMath.MultiplyScalar(p,1/MatrixMath.Square(length_of_t),p);
// 9. Calculate the new residua for the X (spectral) and Y (concentration) matrix
//MatrixMath.MultiplyScalar(t,length_of_t*v,t); // original t times the cross product
MatrixMath.SubtractProductFromSelf(t,p,X);
MatrixMath.MultiplyScalar(t,v,t); // original t times the cross product
MatrixMath.SubtractProductFromSelf(t,q,Y); // to calculate residual Y
// Store the loads of X and Y in the output result matrix
xLoads.AppendBottom(p);
yLoads.AppendBottom(q);
W.AppendBottom(w);
V.AppendRight(v);
if(PRESS!=null)
{
double pressValue=MatrixMath.SumOfSquares(Y);
PRESS.Append(new MatrixMath.Scalar(pressValue));
}
// Calculate SEPcv. If SEPcv is greater than for the actual number of factors,
// break since the optimal number of factors was found. If not, repeat the calculations
// with the residual matrizes for the next factor.
} // for all factors
}
/// <summary>
/// Creates an analyis from preprocessed spectra and preprocessed concentrations.
/// </summary>
/// <param name="matrixX">The spectral matrix (each spectrum is a row in the matrix). They must at least be centered.</param>
/// <param name="matrixY">The matrix of concentrations (each experiment is a row in the matrix). They must at least be centered.</param>
/// <param name="maxFactors">Maximum number of factors for analysis.</param>
/// <returns>A regression object, which holds all the loads and weights neccessary for further calculations.</returns>
protected override void AnalyzeFromPreprocessedWithoutReset(IROMatrix matrixX, IROMatrix matrixY, int maxFactors)
{
int numberOfFactors = _calib.NumberOfFactors = Math.Min(matrixX.Columns, maxFactors);
MatrixMath.BEMatrix _xLoads = new MatrixMath.BEMatrix(0,0);
MatrixMath.BEMatrix _yLoads = new MatrixMath.BEMatrix(0,0);
MatrixMath.BEMatrix _W = new MatrixMath.BEMatrix(0,0);
MatrixMath.REMatrix _V = new MatrixMath.REMatrix(0,0);
_PRESS = VectorMath.CreateExtensibleVector(0);
ExecuteAnalysis(matrixX, matrixY, ref numberOfFactors, _xLoads, _yLoads, _W, _V, _PRESS);
_calib.NumberOfFactors = Math.Min(_calib.NumberOfFactors,numberOfFactors);
_calib.XLoads = _xLoads;
_calib.YLoads = _yLoads;
_calib.XWeights = _W;
_calib.CrossProduct = _V;
}
/// <summary>
/// Using the information in the plsMemo, gets the matrix of original Y (concentration) data.
/// </summary>
/// <param name="plsMemo">The PLS mememto containing the information where to find the original data.</param>
/// <returns>Matrix of orignal Y (concentration) data.</returns>
public static IMatrix GetOriginalY(MultivariateContentMemento plsMemo)
{
string tablename = plsMemo.OriginalDataTableName;
Altaxo.Data.DataTable srctable = Current.Project.DataTableCollection[tablename];
if (srctable == null)
throw new ApplicationException(string.Format("Table[{0}] containing original spectral data not found!", tablename));
Altaxo.Data.DataColumnCollection concentration = plsMemo.SpectrumIsRow ? srctable.DataColumns : srctable.PropertyColumns;
Altaxo.Collections.IAscendingIntegerCollection concentrationIndices = plsMemo.ConcentrationIndices;
Altaxo.Collections.IAscendingIntegerCollection measurementIndices = plsMemo.MeasurementIndices;
// fill in the y-values
MatrixMath.BEMatrix matrixY = new MatrixMath.BEMatrix(measurementIndices.Count, concentrationIndices.Count);
for (int i = 0; i < concentrationIndices.Count; i++)
{
Altaxo.Data.INumericColumn col = concentration[concentrationIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < measurementIndices.Count; j++)
{
matrixY[j, i] = col[measurementIndices[j]];
}
} // end fill in yvalues
return matrixY;
}
/// <summary>
/// Makes a PCA (a principal component analysis) of the table or the selected columns / rows and stores the results in a newly created table.
/// </summary>
/// <param name="mainDocument">The main document of the application.</param>
/// <param name="srctable">The table where the data come from.</param>
/// <param name="selectedColumns">The selected columns.</param>
/// <param name="selectedRows">The selected rows.</param>
/// <param name="bHorizontalOrientedSpectrum">True if a spectrum is a single row, False if a spectrum is a single column.</param>
/// <param name="maxNumberOfFactors">The maximum number of factors to calculate.</param>
/// <returns></returns>
public static string PrincipalComponentAnalysis(
Altaxo.AltaxoDocument mainDocument,
Altaxo.Data.DataTable srctable,
IAscendingIntegerCollection selectedColumns,
IAscendingIntegerCollection selectedRows,
bool bHorizontalOrientedSpectrum,
int maxNumberOfFactors
)
{
bool bUseSelectedColumns = (null!=selectedColumns && 0!=selectedColumns.Count);
int prenumcols = bUseSelectedColumns ? selectedColumns.Count : srctable.DataColumns.ColumnCount;
// check for the number of numeric columns
int numcols = 0;
for(int i=0;i<prenumcols;i++)
{
int idx = bUseSelectedColumns ? selectedColumns[i] : i;
if(srctable[i] is Altaxo.Data.INumericColumn)
numcols++;
}
// check the number of rows
bool bUseSelectedRows = (null!=selectedRows && 0!=selectedRows.Count);
int numrows;
if(bUseSelectedRows)
numrows = selectedRows.Count;
else
{
numrows = 0;
for(int i=0;i<numcols;i++)
{
int idx = bUseSelectedColumns ? selectedColumns[i] : i;
numrows = Math.Max(numrows,srctable[idx].Count);
}
}
// check that both dimensions are at least 2 - otherwise PCA is not possible
if(numrows<2)
return "At least two rows are neccessary to do Principal Component Analysis!";
if(numcols<2)
return "At least two numeric columns are neccessary to do Principal Component Analysis!";
// Create a matrix of appropriate dimensions and fill it
MatrixMath.BEMatrix matrixX;
if(bHorizontalOrientedSpectrum)
{
matrixX = new MatrixMath.BEMatrix(numrows,numcols);
int ccol = 0; // current column in the matrix
for(int i=0;i<prenumcols;i++)
{
int colidx = bUseSelectedColumns ? selectedColumns[i] : i;
Altaxo.Data.INumericColumn col = srctable[colidx] as Altaxo.Data.INumericColumn;
if(null!=col)
{
for(int j=0;j<numrows;j++)
{
int rowidx = bUseSelectedRows ? selectedRows[j] : j;
matrixX[j,ccol] = col[rowidx];
}
++ccol;
}
}
} // end if it was a horizontal oriented spectrum
else // if it is a vertical oriented spectrum
{
matrixX = new MatrixMath.BEMatrix(numcols,numrows);
int ccol = 0; // current column in the matrix
for(int i=0;i<prenumcols;i++)
{
int colidx = bUseSelectedColumns ? selectedColumns[i] : i;
Altaxo.Data.INumericColumn col = srctable[colidx] as Altaxo.Data.INumericColumn;
if(null!=col)
{
for(int j=0;j<numrows;j++)
{
int rowidx = bUseSelectedRows ? selectedRows[j] : j;
matrixX[ccol,j] = col[rowidx];
}
++ccol;
}
}
} // if it was a vertical oriented spectrum
// now do PCA with the matrix
MatrixMath.REMatrix factors = new MatrixMath.REMatrix(0,0);
MatrixMath.BEMatrix loads = new MatrixMath.BEMatrix(0,0);
MatrixMath.BEMatrix residualVariances = new MatrixMath.BEMatrix(0,0);
MatrixMath.HorizontalVector meanX = new MatrixMath.HorizontalVector(matrixX.Columns);
// first, center the matrix
MatrixMath.ColumnsToZeroMean(matrixX,meanX);
MatrixMath.NIPALS_HO(matrixX,maxNumberOfFactors,1E-9,factors,loads,residualVariances);
// now we have to create a new table where to place the calculated factors and loads
// we will do that in a vertical oriented manner, i.e. even if the loads are
// here in horizontal vectors: in our table they are stored in (vertical) columns
Altaxo.Data.DataTable table = new Altaxo.Data.DataTable("PCA of " + srctable.Name);
// Fill the Table
table.Suspend();
// first of all store the meanscore
{
double meanScore = MatrixMath.LengthOf(meanX);
MatrixMath.NormalizeRows(meanX);
Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn();
for(int i=0;i<factors.Rows;i++)
col[i] = meanScore;
table.DataColumns.Add(col,"MeanFactor",Altaxo.Data.ColumnKind.V,0);
}
// first store the factors
for(int i=0;i<factors.Columns;i++)
{
Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn();
for(int j=0;j<factors.Rows;j++)
col[j] = factors[j,i];
table.DataColumns.Add(col,"Factor"+i.ToString(),Altaxo.Data.ColumnKind.V,1);
}
// now store the mean of the matrix
{
Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn();
for(int j=0;j<meanX.Columns;j++)
col[j] = meanX[0,j];
table.DataColumns.Add(col,"MeanLoad",Altaxo.Data.ColumnKind.V,2);
}
// now store the loads - careful - they are horizontal in the matrix
for(int i=0;i<loads.Rows;i++)
{
Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn();
for(int j=0;j<loads.Columns;j++)
col[j] = loads[i,j];
table.DataColumns.Add(col,"Load"+i.ToString(),Altaxo.Data.ColumnKind.V,3);
}
// now store the residual variances, they are vertical in the vector
{
Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn();
for(int i=0;i<residualVariances.Rows;i++)
col[i] = residualVariances[i,0];
table.DataColumns.Add(col,"ResidualVariance",Altaxo.Data.ColumnKind.V,4);
}
table.Resume();
mainDocument.DataTableCollection.Add(table);
// create a new worksheet without any columns
Current.ProjectService.CreateNewWorksheet(table);
return null;
}
/// <summary>
/// This predicts concentrations of unknown spectra.
/// </summary>
/// <param name="XU">Matrix of unknown spectra (preprocessed the same way as the calibration spectra).</param>
/// <param name="numFactors">Number of factors used for prediction.</param>
/// <returns>The predicted y values. (They are centered).</returns>
public virtual IROMatrix PredictYFromPreprocessed(
IROMatrix XU, // unknown spectrum or spectra, horizontal oriented
int numFactors // number of factors to use for prediction
)
{
IMatrix predictedY = new MatrixMath.BEMatrix(XU.Rows,InternalCalibrationModel.NumberOfY);
this.PredictedYAndSpectralResidualsFromPreprocessed(XU,numFactors,predictedY,null);
return predictedY;
}
/// <summary>
/// This calculates the spectral residuals.
/// </summary>
/// <param name="XU">Spectra (horizontally oriented).</param>
/// <param name="numFactors">Number of factors used for calculation.</param>
/// <returns>The calculated spectral residuals.</returns>
public virtual IROMatrix SpectralResidualsFromPreprocessed(
IROMatrix XU, // unknown spectrum or spectra, horizontal oriented
int numFactors // number of factors to use for prediction
)
{
IMatrix result = new MatrixMath.BEMatrix(XU.Rows,this.NumberOfSpectralResiduals);
SpectralResidualsFromPreprocessed(XU,numFactors,result);
return result;
}
/// <summary>
/// Creates an analyis from preprocessed spectra and preprocessed concentrations.
/// </summary>
/// <param name="matrixX">The spectral matrix (each spectrum is a row in the matrix). They must at least be centered.</param>
/// <param name="matrixY">The matrix of concentrations (each experiment is a row in the matrix). They must at least be centered.</param>
/// <param name="maxFactors">Maximum number of factors for analysis.</param>
/// <returns>A regression object, which holds all the loads and weights neccessary for further calculations.</returns>
protected override void AnalyzeFromPreprocessedWithoutReset(IROMatrix matrixX, IROMatrix matrixY, int maxFactors)
{
int numFactors = Math.Min(matrixX.Columns, maxFactors);
IROMatrix xLoads, xScores;
IROVector V;
ExecuteAnalysis(matrixX, matrixY, ref numFactors, out xLoads, out xScores, out V);
IMatrix yLoads = new MatrixMath.BEMatrix(matrixY.Rows,matrixY.Columns);
MatrixMath.Copy(matrixY,yLoads);
_calib.NumberOfFactors = numFactors;
_calib.XLoads = xLoads;
_calib.YLoads = yLoads;
_calib.XScores = xScores;
_calib.CrossProduct = V;
}
/// <summary>
/// Calculates the prediction scores (for use withthe preprocessed spectra).
/// </summary>
/// <param name="numFactors">Number of factors used to calculate the prediction scores.</param>
/// <param name="predictionScores">Supplied matrix for holding the prediction scores.</param>
protected override void InternalGetPredictionScores(int numFactors, IMatrix predictionScores)
{
IMatrix pred = new MatrixMath.BEMatrix(predictionScores.Rows, 1);
for (int i = 0; i < _calib.NumberOfY; i++)
{
PLS2Regression.GetPredictionScoreMatrix(_calib.XLoads[i], _calib.YLoads[i], _calib.XWeights[i], _calib.CrossProduct[i], numFactors, pred);
MatrixMath.SetColumn(pred, predictionScores, i);
}
}
public static void CalculateXLeverageFromPreprocessed(
IROMatrix xScores,
int numberOfFactors,
IMatrix leverage)
{
IMatrix subscores = new MatrixMath.BEMatrix(xScores.Rows,numberOfFactors);
MatrixMath.Submatrix(xScores,subscores);
MatrixMath.SingularValueDecomposition decompose = new MatrixMath.SingularValueDecomposition(subscores);
for(int i=0;i<xScores.Rows;i++)
leverage[i,0] = decompose.HatDiagonal[i];
}
public static void GetSpectralResiduals(
IROMatrix matrixX,
IROMatrix xLoads,
IROMatrix yLoads,
IROMatrix xScores,
IROVector crossProduct,
int numberOfFactors,
IMatrix spectralResiduals)
{
int numX = xLoads.Columns;
int numY = yLoads.Columns;
int numM = yLoads.Rows;
MatrixMath.BEMatrix reconstructedSpectra = new MatrixMath.BEMatrix(matrixX.Rows,matrixX.Columns);
MatrixMath.ZeroMatrix(reconstructedSpectra);
for(int nf=0;nf<numberOfFactors;nf++)
{
double scale = crossProduct[nf];
for(int m=0;m<numM;m++)
{
for(int k=0;k<numX;k++)
reconstructedSpectra[m,k] += scale*xScores[m,nf]*xLoads[nf,k];
}
}
for(int m=0;m<numM;m++)
spectralResiduals[m,0] = MatrixMath.SumOfSquaredDifferences(
MatrixMath.ToROSubMatrix(matrixX,m,0,1,matrixX.Columns),
MatrixMath.ToROSubMatrix(reconstructedSpectra,m,0,1,matrixX.Columns));
}
public static void CalculatePRESS(
IROMatrix yLoads,
IROMatrix xScores,
int numberOfFactors,
out IROVector press)
{
int numMeasurements = yLoads.Rows;
IExtensibleVector PRESS = VectorMath.CreateExtensibleVector(numberOfFactors+1);
MatrixMath.BEMatrix UtY = new MatrixMath.BEMatrix(yLoads.Rows,yLoads.Columns);
MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(yLoads.Rows,yLoads.Columns);
press = PRESS;
MatrixMath.MultiplyFirstTransposed(xScores,yLoads,UtY);
// now calculate PRESS by predicting the y
// using yp = U (w*(1/w)) U' y
// of course w*1/w is the identity matrix, but we use only the first factors, so using a cutted identity matrix
// we precalculate the last term U'y = UtY
// and multiplying with one row of U in every factor step, summing up the predictedY
PRESS[0] = MatrixMath.SumOfSquares(yLoads);
for(int nf=0;nf<numberOfFactors;nf++)
{
for(int cn=0;cn<yLoads.Columns;cn++)
{
for(int k=0;k<yLoads.Rows;k++)
predictedY[k,cn] += xScores[k,nf]*UtY[nf,cn];
}
PRESS[nf+1] = MatrixMath.SumOfSquaredDifferences(yLoads,predictedY);
}
}
/// <summary>
/// Using the information in the plsMemo, gets the matrix of original spectra. The spectra are horizontal in the matrix, i.e. each spectra is a matrix row.
/// </summary>
/// <param name="plsMemo">The PLS memento containing the information about the location of the original data.</param>
/// <returns>The matrix of the original spectra.</returns>
public static IMatrix GetRawSpectra(MultivariateContentMemento plsMemo)
{
string tablename = plsMemo.OriginalDataTableName;
if (!Current.Project.DataTableCollection.Contains(tablename))
throw new OriginalDataTableNotFoundException(string.Format("The original data table <<{0}>> does not exist, was it renamed?", tablename));
Altaxo.Data.DataTable srctable = Current.Project.DataTableCollection[tablename];
if (srctable == null)
throw new ApplicationException(string.Format("Table[{0}] containing original spectral data not found!", tablename));
Altaxo.Collections.IAscendingIntegerCollection spectralIndices = plsMemo.SpectralIndices;
Altaxo.Collections.IAscendingIntegerCollection measurementIndices = plsMemo.MeasurementIndices;
MatrixMath.BEMatrix matrixX =
new MatrixMath.BEMatrix(measurementIndices.Count, spectralIndices.Count);
return GetRawSpectra(srctable, plsMemo.SpectrumIsRow, spectralIndices, measurementIndices);
}
/// <summary>
/// This calculates the spectral residuals. The matrices are reallocated if they don't have the appropriate dimensions.
/// </summary>
/// <param name="XU">Spectra (horizontally oriented).</param>
/// <param name="numFactors">Number of factors used for calculation.</param>
/// <param name="predictedY">On return, holds the predicted y values. (They are centered). If the matrix you provide has not the appropriate dimensions, it is reallocated.</param>
/// <param name="spectralResiduals">On return, holds the spectral residual values. If the matrix you provide has not the appropriate dimensions, it is reallocated.</param>
public virtual void PredictedYAndSpectralResidualsFromPreprocessed(
IROMatrix XU, // unknown spectrum or spectra, horizontal oriented
int numFactors, // number of factors to use for prediction
ref IMatrix predictedY,
ref IMatrix spectralResiduals // Matrix of spectral residuals, n rows x 1 column
)
{
// check the dimensions of the matrices
if(predictedY!=null)
{
if(predictedY.Rows!=XU.Rows || predictedY.Columns != this.InternalCalibrationModel.NumberOfY)
predictedY = new MatrixMath.BEMatrix(XU.Rows,InternalCalibrationModel.NumberOfY);
}
if(spectralResiduals!=null)
{
if(spectralResiduals.Rows!=XU.Rows || spectralResiduals.Columns != this.NumberOfSpectralResiduals)
spectralResiduals = new MatrixMath.BEMatrix(XU.Rows,this.NumberOfSpectralResiduals);
}
PredictedYAndSpectralResidualsFromPreprocessed(XU,numFactors,predictedY,spectralResiduals);
}
/// <summary>
/// Fills a matrix with the selected data of a table.
/// </summary>
/// <param name="srctable">The source table where the data for the spectra are located.</param>
/// <param name="spectrumIsRow">True if the spectra in the table are organized horizontally, false if spectra are vertically oriented.</param>
/// <param name="spectralIndices">The selected indices wich indicate all (wavelength, frequencies, etc.) that belong to one spectrum. If spectrumIsRow==true, this are the selected column indices, otherwise the selected row indices.</param>
/// <param name="measurementIndices">The indices of all measurements (spectra) selected.</param>
/// <returns>The matrix of spectra. In this matrix the spectra are horizonally organized (each row is one spectrum).</returns>
public static IMatrix GetRawSpectra(Altaxo.Data.DataTable srctable, bool spectrumIsRow, Altaxo.Collections.IAscendingIntegerCollection spectralIndices, Altaxo.Collections.IAscendingIntegerCollection measurementIndices)
{
if (srctable == null)
throw new ArgumentException("Argument srctable may not be null");
MatrixMath.BEMatrix matrixX =
new MatrixMath.BEMatrix(measurementIndices.Count, spectralIndices.Count);
if (spectrumIsRow)
{
for (int i = 0; i < spectralIndices.Count; i++)
{
// labelColumnOfX[i] = spectralIndices[i];
Altaxo.Data.INumericColumn col = srctable[spectralIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < measurementIndices.Count; j++)
{
matrixX[j, i] = col[measurementIndices[j]];
}
} // end fill in x-values
}
else // vertical oriented spectrum
{
for (int i = 0; i < spectralIndices.Count; i++)
{
// labelColumnOfX[i] = spectralIndices[i];
}
for (int i = 0; i < measurementIndices.Count; i++)
{
Altaxo.Data.INumericColumn col = srctable[measurementIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < spectralIndices.Count; j++)
{
matrixX[i, j] = col[spectralIndices[j]];
}
} // end fill in x-values
}
return matrixX;
}
/// <summary>
/// This calculates the spectral residuals. The matrices are reallocated if they don't have the appropriate dimensions.
/// </summary>
/// <param name="XU">Spectra (horizontally oriented).</param>
/// <param name="numFactors">Number of factors used for calculation.</param>
/// <param name="calculatePredictedY">If true, the predictedY is calculated. Otherwise, predictedY is null on return.</param>
/// <param name="predictedY">On return, holds the predicted y values. (They are centered). If the matrix you provide has not the appropriate dimensions, it is reallocated.</param>
/// <param name="calculateSpectralResiduals">If true, the spectral residuals are calculated. Otherwise spectralResiduals is null on return.</param>
/// <param name="spectralResiduals">On return, holds the spectral residual values. If the matrix you provide has not the appropriate dimensions, it is reallocated.</param>
public virtual void PredictedYAndSpectralResidualsFromPreprocessed(
IROMatrix XU, // unknown spectrum or spectra, horizontal oriented
int numFactors, // number of factors to use for prediction
bool calculatePredictedY,
out IMatrix predictedY,
bool calculateSpectralResiduals,
out IMatrix spectralResiduals // Matrix of spectral residuals, n rows x 1 column
)
{
// check the dimensions of the matrices
if(calculatePredictedY)
predictedY = new MatrixMath.BEMatrix(XU.Rows,InternalCalibrationModel.NumberOfY);
else
predictedY=null;
if(calculateSpectralResiduals)
spectralResiduals = new MatrixMath.BEMatrix(XU.Rows,this.NumberOfSpectralResiduals);
else
spectralResiduals = null;
PredictedYAndSpectralResidualsFromPreprocessed(XU,numFactors,predictedY,spectralResiduals);
}
/// <summary>
/// Get the matrix of x and y values (raw data).
/// </summary>
/// <param name="srctable">The table where the data come from.</param>
/// <param name="selectedColumns">The selected columns.</param>
/// <param name="selectedRows">The selected rows.</param>
/// <param name="selectedPropertyColumns">The selected property column(s).</param>
/// <param name="bHorizontalOrientedSpectrum">True if a spectrum is a single row, False if a spectrum is a single column.</param>
/// <param name="matrixX">On return, gives the matrix of spectra (each spectra is a row in the matrix).</param>
/// <param name="matrixY">On return, gives the matrix of y-values (each measurement is a row in the matrix).</param>
/// <param name="plsContent">Holds information about the analysis results.</param>
/// <param name="xOfX">On return, this is the vector of values corresponding to each spectral bin, i.e. wavelength values, frequencies etc.</param>
/// <returns></returns>
public static string GetXYMatrices(
Altaxo.Data.DataTable srctable,
IAscendingIntegerCollection selectedColumns,
IAscendingIntegerCollection selectedRows,
IAscendingIntegerCollection selectedPropertyColumns,
bool bHorizontalOrientedSpectrum,
MultivariateContentMemento plsContent,
out IMatrix matrixX,
out IMatrix matrixY,
out IROVector xOfX
)
{
matrixX = null;
matrixY = null;
xOfX = null;
plsContent.SpectrumIsRow = bHorizontalOrientedSpectrum;
Altaxo.Data.DataColumn xColumnOfX = null;
Altaxo.Data.DataColumn labelColumnOfX = new Altaxo.Data.DoubleColumn();
Altaxo.Data.DataColumnCollection concentration = bHorizontalOrientedSpectrum ? srctable.DataColumns : srctable.PropertyColumns;
// we presume for now that the spectrum is horizontally,
// if not we exchange the collections later
AscendingIntegerCollection numericDataCols = new AscendingIntegerCollection();
AscendingIntegerCollection numericDataRows = new AscendingIntegerCollection();
AscendingIntegerCollection concentrationIndices = new AscendingIntegerCollection();
AscendingIntegerCollection spectralIndices = bHorizontalOrientedSpectrum ? numericDataCols : numericDataRows;
AscendingIntegerCollection measurementIndices = bHorizontalOrientedSpectrum ? numericDataRows : numericDataCols;
plsContent.ConcentrationIndices = concentrationIndices;
plsContent.MeasurementIndices = measurementIndices;
plsContent.SpectralIndices = spectralIndices;
plsContent.SpectrumIsRow = bHorizontalOrientedSpectrum;
plsContent.OriginalDataTableName = srctable.Name;
bool bUseSelectedColumns = (null != selectedColumns && 0 != selectedColumns.Count);
// this is the number of columns (for now), but it can be less than this in case
// not all columns are numeric
int prenumcols = bUseSelectedColumns ? selectedColumns.Count : srctable.DataColumns.ColumnCount;
// check for the number of numeric columns
int numcols = 0;
for (int i = 0; i < prenumcols; i++)
{
int idx = bUseSelectedColumns ? selectedColumns[i] : i;
if (srctable[idx] is Altaxo.Data.INumericColumn)
{
numericDataCols.Add(idx);
numcols++;
}
}
// check the number of rows
bool bUseSelectedRows = (null != selectedRows && 0 != selectedRows.Count);
int numrows;
if (bUseSelectedRows)
{
numrows = selectedRows.Count;
numericDataRows.Add(selectedRows);
}
else
{
numrows = 0;
for (int i = 0; i < numcols; i++)
{
int idx = bUseSelectedColumns ? selectedColumns[i] : i;
numrows = Math.Max(numrows, srctable[idx].Count);
}
numericDataRows.Add(ContiguousIntegerRange.FromStartAndCount(0, numrows));
}
if (bHorizontalOrientedSpectrum)
{
if (numcols < 2)
return "At least two numeric columns are neccessary to do Partial Least Squares (PLS) analysis!";
// check that the selected columns are in exactly two groups
// the group which has more columns is then considered to have
// the spectrum, the other group is the y-values
int group0 = -1;
int group1 = -1;
int groupcount0 = 0;
int groupcount1 = 0;
for (int i = 0; i < numcols; i++)
{
int grp = srctable.DataColumns.GetColumnGroup(numericDataCols[i]);
if (group0 < 0)
{
group0 = grp;
groupcount0 = 1;
}
else if (group0 == grp)
{
groupcount0++;
}
else if (group1 < 0)
{
group1 = grp;
groupcount1 = 1;
}
else if (group1 == grp)
{
groupcount1++;
}
else
{
return "The columns you selected must be members of two groups (y-values and spectrum), but actually there are more than two groups!";
}
} // end for all columns
if (groupcount1 <= 0)
return "The columns you selected must be members of two groups (y-values and spectrum), but actually only one group was detected!";
if (groupcount1 < groupcount0)
{
int hlp;
hlp = groupcount1;
groupcount1 = groupcount0;
groupcount0 = hlp;
hlp = group1;
group1 = group0;
group0 = hlp;
}
// group0 is now the group of y-values (concentrations)
// group1 is now the group of x-values (spectra)
// we delete group0 from numericDataCols and add it to concentrationIndices
for (int i = numcols - 1; i >= 0; i--)
{
int index = numericDataCols[i];
if (group0 == srctable.DataColumns.GetColumnGroup(index))
{
numericDataCols.Remove(index);
concentrationIndices.Add(index);
}
}
// fill the corresponding X-Column of the spectra
xColumnOfX = Altaxo.Data.DataColumn.CreateColumnOfSelectedRows(
srctable.PropertyColumns.FindXColumnOfGroup(group1),
spectralIndices);
}
else // vertically oriented spectrum -> one spectrum is one data column
{
// we have to exchange measurementIndices and
// if PLS on columns, than we should have property columns selected
// that designates the y-values
// so count all property columns
bool bUseSelectedPropCols = (null != selectedPropertyColumns && 0 != selectedPropertyColumns.Count);
// this is the number of property columns (for now), but it can be less than this in case
// not all columns are numeric
int prenumpropcols = bUseSelectedPropCols ? selectedPropertyColumns.Count : srctable.PropCols.ColumnCount;
// check for the number of numeric property columns
for (int i = 0; i < prenumpropcols; i++)
{
int idx = bUseSelectedPropCols ? selectedPropertyColumns[i] : i;
if (srctable.PropCols[idx] is Altaxo.Data.INumericColumn)
{
concentrationIndices.Add(idx);
}
}
if (concentrationIndices.Count < 1)
return "At least one numeric property column must exist to hold the y-values!";
// fill the corresponding X-Column of the spectra
xColumnOfX = Altaxo.Data.DataColumn.CreateColumnOfSelectedRows(
srctable.DataColumns.FindXColumnOf(srctable[measurementIndices[0]]), spectralIndices);
} // else vertically oriented spectrum
IVector xOfXRW = VectorMath.CreateExtensibleVector(xColumnOfX.Count);
xOfX = xOfXRW;
if (xColumnOfX is INumericColumn)
{
for (int i = 0; i < xOfX.Length; i++)
xOfXRW[i] = ((INumericColumn)xColumnOfX)[i];
}
else
{
for (int i = 0; i < xOfX.Length; i++)
xOfXRW[i] = spectralIndices[i];
}
// now fill the matrix
// fill in the y-values
matrixY = new MatrixMath.BEMatrix(measurementIndices.Count, concentrationIndices.Count);
for (int i = 0; i < concentrationIndices.Count; i++)
{
Altaxo.Data.INumericColumn col = concentration[concentrationIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < measurementIndices.Count; j++)
{
matrixY[j, i] = col[measurementIndices[j]];
}
} // end fill in yvalues
matrixX = new MatrixMath.BEMatrix(measurementIndices.Count, spectralIndices.Count);
if (bHorizontalOrientedSpectrum)
{
for (int i = 0; i < spectralIndices.Count; i++)
{
labelColumnOfX[i] = spectralIndices[i];
Altaxo.Data.INumericColumn col = srctable[spectralIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < measurementIndices.Count; j++)
{
matrixX[j, i] = col[measurementIndices[j]];
}
} // end fill in x-values
}
else // vertical oriented spectrum
{
for (int i = 0; i < spectralIndices.Count; i++)
{
labelColumnOfX[i] = spectralIndices[i];
}
for (int i = 0; i < measurementIndices.Count; i++)
{
Altaxo.Data.INumericColumn col = srctable[measurementIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < spectralIndices.Count; j++)
{
matrixX[i, j] = col[spectralIndices[j]];
}
} // end fill in x-values
}
return null;
}
/// <summary>
/// This function separates the spectra into a bunch of spectra used for calibration and the rest of spectra
/// used for prediction. This separation is repeated until all spectra are used exactly one time for prediction.
/// </summary>
/// <param name="X">Matrix of spectra (horizontal oriented).</param>
/// <param name="Y">Matrix of y values.</param>
/// <param name="groupingStrategy">The strategy how to separate the spectra into the calibration and prediction spectra.</param>
/// <param name="crossFunction">The function that is called for each separation.</param>
/// <returns>The mean number of spectra that was used for prediction.</returns>
public static double CrossValidationIteration(
IROMatrix X, // matrix of spectra (a spectra is a row of this matrix)
IROMatrix Y, // matrix of concentrations (a mixture is a row of this matrix)
ICrossValidationGroupingStrategy groupingStrategy,
CrossValidationIterationFunction crossFunction
)
{
// int[][] groups = bExcludeGroups ? new ExcludeGroupsGroupingStrategy().Group(Y) : new ExcludeSingleMeasurementsGroupingStrategy().Group(Y);
int[][] groups = groupingStrategy.Group(Y);
IMatrix XX=null;
IMatrix YY=null;
IMatrix XU=null;
IMatrix YU=null;
for(int nGroup=0 ,prevNumExcludedSpectra = int.MinValue ;nGroup < groups.Length;nGroup++)
{
int[] spectralGroup = groups[nGroup];
int numberOfExcludedSpectraOfGroup = spectralGroup.Length;
if(prevNumExcludedSpectra != numberOfExcludedSpectraOfGroup)
{
XX = new MatrixMath.BEMatrix(X.Rows-numberOfExcludedSpectraOfGroup,X.Columns);
YY = new MatrixMath.BEMatrix(Y.Rows-numberOfExcludedSpectraOfGroup,Y.Columns);
XU = new MatrixMath.BEMatrix(numberOfExcludedSpectraOfGroup,X.Columns);
YU = new MatrixMath.BEMatrix(numberOfExcludedSpectraOfGroup,Y.Columns);
prevNumExcludedSpectra = numberOfExcludedSpectraOfGroup;
}
// build a new x and y matrix with the group information
// fill XX and YY with values
for(int i=0,j=0;i<X.Rows;i++)
{
if(Array.IndexOf(spectralGroup,i)>=0) // if spectral group contains i
continue; // Exclude this row from the spectra
MatrixMath.SetRow(X,i,XX,j);
MatrixMath.SetRow(Y,i,YY,j);
j++;
}
// fill XU (unknown spectra) with values
for(int i=0;i<spectralGroup.Length;i++)
{
int j = spectralGroup[i];
MatrixMath.SetRow(X,j,XU,i); // x-unkown (unknown spectra)
MatrixMath.SetRow(Y,j,YU,i); // y-unkown (unknown concentration)
}
// now do the analysis
crossFunction(spectralGroup,XX,YY,XU,YU);
} // for all groups
// calculate the mean number of excluded spectras
return ((double)X.Rows)/groups.Length;
}
/// <summary>
/// Multiplies selected columns to form a matrix.
/// </summary>
/// <param name="mainDocument"></param>
/// <param name="srctable"></param>
/// <param name="selectedColumns"></param>
/// <returns>Null if successful, else the description of the error.</returns>
/// <remarks>The user must select an even number of columns. All columns of the first half of the selection
/// must have the same number of rows, and all columns of the second half of selection must also have the same
/// number of rows. The first half of selected columns form a matrix of dimensions(firstrowcount,halfselected), and the second half
/// of selected columns form a matrix of dimension(halfselected, secondrowcount). The resulting matrix has dimensions (firstrowcount,secondrowcount) and is
/// stored in a separate worksheet.</remarks>
public static string MultiplyColumnsToMatrix(
Altaxo.AltaxoDocument mainDocument,
Altaxo.Data.DataTable srctable,
IAscendingIntegerCollection selectedColumns
)
{
// check that there are columns selected
if(0==selectedColumns.Count)
return "You must select at least two columns to multiply!";
// selected columns must contain an even number of columns
if(0!=selectedColumns.Count%2)
return "You selected an odd number of columns. Please select an even number of columns to multiply!";
// all selected columns must be numeric columns
for(int i=0;i<selectedColumns.Count;i++)
{
if(!(srctable[selectedColumns[i]] is Altaxo.Data.INumericColumn))
return string.Format("The column[{0}] (name:{1}) is not a numeric column!",selectedColumns[i],srctable[selectedColumns[i]].Name);
}
int halfselect = selectedColumns.Count/2;
// check that all columns from the first half of selected colums contain the same
// number of rows
int rowsfirsthalf=int.MinValue;
for(int i=0;i<halfselect;i++)
{
int idx = selectedColumns[i];
if(rowsfirsthalf<0)
rowsfirsthalf = srctable[idx].Count;
else if(rowsfirsthalf != srctable[idx].Count)
return "The first half of selected columns have not all the same length!";
}
int rowssecondhalf=int.MinValue;
for(int i=halfselect;i<selectedColumns.Count;i++)
{
int idx = selectedColumns[i];
if(rowssecondhalf<0)
rowssecondhalf = srctable[idx].Count;
else if(rowssecondhalf != srctable[idx].Count)
return "The second half of selected columns have not all the same length!";
}
// now create the matrices to multiply from the
MatrixMath.REMatrix firstMat = new MatrixMath.REMatrix(rowsfirsthalf,halfselect);
for(int i=0;i<halfselect;i++)
{
Altaxo.Data.INumericColumn col = (Altaxo.Data.INumericColumn)srctable[selectedColumns[i]];
for(int j=0;j<rowsfirsthalf;j++)
firstMat[j,i] = col[j];
}
MatrixMath.BEMatrix secondMat = new MatrixMath.BEMatrix(halfselect,rowssecondhalf);
for(int i=0;i<halfselect;i++)
{
Altaxo.Data.INumericColumn col = (Altaxo.Data.INumericColumn)srctable[selectedColumns[i+halfselect]];
for(int j=0;j<rowssecondhalf;j++)
secondMat[i,j] = col[j];
}
// now multiply the two matrices
MatrixMath.BEMatrix resultMat = new MatrixMath.BEMatrix(rowsfirsthalf,rowssecondhalf);
MatrixMath.Multiply(firstMat,secondMat,resultMat);
// and store the result in a new worksheet
Altaxo.Data.DataTable table = new Altaxo.Data.DataTable("ResultMatrix of " + srctable.Name);
table.Suspend();
// first store the factors
for(int i=0;i<resultMat.Columns;i++)
{
Altaxo.Data.DoubleColumn col = new Altaxo.Data.DoubleColumn();
for(int j=0;j<resultMat.Rows;j++)
col[j] = resultMat[j,i];
table.DataColumns.Add(col,i.ToString());
}
table.Resume();
mainDocument.DataTableCollection.Add(table);
// create a new worksheet without any columns
Current.ProjectService.CreateNewWorksheet(table);
return null;
}
/// <summary>
/// Calculates the prediction scores.
/// </summary>
/// <param name="numberOfFactors">Number of factors used for calculation.</param>
/// <returns>The prediction score matrix. This matrix has the dimensions (NumberOfX, NumberOfY).</returns>
public virtual IROMatrix GetPredictionScores(int numberOfFactors)
{
IMatrix result = new MatrixMath.BEMatrix(InternalCalibrationModel.NumberOfX,InternalCalibrationModel.NumberOfY);
this.InternalGetPredictionScores(numberOfFactors,result);
return result;
}
} // end partial-least-squares-predict
public static void CalculateXLeverageFromPreprocessed(
IROMatrix matrixX,
IROMatrix W, // weighting matrix
int numFactors, // number of factors to use for prediction
IMatrix leverage, // Matrix of predicted y-values, must be same number of rows as spectra
int leverageColumn
)
{
// get the score matrix
MatrixMath.BEMatrix weights = new MatrixMath.BEMatrix(numFactors,W.Columns);
MatrixMath.Submatrix(W,weights,0,0);
MatrixMath.BEMatrix scoresMatrix = new MatrixMath.BEMatrix(matrixX.Rows,weights.Rows);
MatrixMath.MultiplySecondTransposed(matrixX,weights,scoresMatrix);
MatrixMath.SingularValueDecomposition decomposition = MatrixMath.GetSingularValueDecomposition(scoresMatrix);
for(int i=0;i<matrixX.Rows;i++)
leverage[i,leverageColumn] = decomposition.HatDiagonal[i];
}
static void CalculatePRESS(
IROMatrix Y, // matrix of concentrations (a mixture is a row of this matrix)
IROMatrix xLoads, // out: the loads of the X matrix
IROMatrix xScores, // matrix of weighting values
IROVector V, // vector of cross products
int maxNumberOfFactors,
IVector PRESS //vector of Y PRESS values
)
{
IROMatrix U = xScores;
MatrixMath.BEMatrix UtY = new MatrixMath.BEMatrix(Y.Rows,Y.Columns);
MatrixMath.MultiplyFirstTransposed(U,Y,UtY);
MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(Y.Rows,Y.Columns);
MatrixMath.BEMatrix subU = new MatrixMath.BEMatrix(Y.Rows,1);
MatrixMath.BEMatrix subY = new MatrixMath.BEMatrix(Y.Rows,Y.Columns);
PRESS[0] = MatrixMath.SumOfSquares(Y);
int numFactors = Math.Min(maxNumberOfFactors,V.Length);
// now calculate PRESS by predicting the y
// using yp = U (w*(1/w)) U' y
// of course w*1/w is the identity matrix, but we use only the first factors, so using a cutted identity matrix
// we precalculate the last term U'y = UtY
// and multiplying with one row of U in every factor step, summing up the predictedY
for(int nf=0;nf<numFactors;nf++)
{
for(int cn=0;cn<Y.Columns;cn++)
{
for(int k=0;k<Y.Rows;k++)
predictedY[k,cn] += U[k,nf]*UtY[nf,cn];
}
PRESS[nf+1] = MatrixMath.SumOfSquaredDifferences(Y,predictedY);
}
}
/// <summary>
/// Fits a data set linear to a given x base.
/// </summary>
/// <param name="xbase">The matrix of x values of the data set. Dimensions: numberOfData x numberOfParameters. The matrix is changed during calculation!</param>
/// <param name="yarr">The array of y values of the data set.</param>
/// <param name="stddev">The array of y standard deviations of the data set. Can be null if the standard deviation is unkown.</param>
/// <param name="numberOfData">The number of data points (may be smaller than the array sizes of the data arrays).</param>
/// <param name="numberOfParameter">The number of parameters to fit == size of the function base.</param>
/// <param name="threshold">A treshold value (usually 1E-5) used to chop the unimportant singular values away.</param>
public LinearFitBySvd Calculate(
IROMatrix xbase, // NumberOfData, NumberOfParameters
double[] yarr,
double[] stddev,
int numberOfData,
int numberOfParameter,
double threshold)
{
_numberOfParameter = numberOfParameter;
_numberOfFreeParameter = numberOfParameter;
_numberOfData = numberOfData;
_parameter = new double[numberOfParameter];
_residual = new double[numberOfData];
_predicted = new double[numberOfData];
_reducedPredictionVariance = new double[numberOfData];
double[] scaledY = new double[numberOfData];
// Calculated some useful values
_yMean = Mean(yarr,0,_numberOfData);
_yCorrectedSumOfSquares = CorrectedSumOfSquares(yarr,_yMean,0,_numberOfData);
MatrixMath.BEMatrix u = new MatrixMath.BEMatrix(numberOfData,numberOfParameter);
// Fill the function base matrix (rows: numberOfData, columns: numberOfParameter)
// and scale also y
if (null == stddev)
{
for (int i = 0; i < numberOfData; i++)
{
for (int j = 0; j < numberOfParameter; j++)
u[i, j] = xbase[i, j];
scaledY[i] = yarr[i];
}
}
else
{
for (int i = 0; i < numberOfData; i++)
{
double scale = 1 / stddev[i];
for (int j = 0; j < numberOfParameter; j++)
u[i, j] = scale * xbase[i, j];
scaledY[i] = scale * yarr[i];
}
}
_decomposition = MatrixMath.GetSingularValueDecomposition(u);
// set singular values < thresholdLevel to zero
// ChopSingularValues makes only sense if all columns of the x matrix have the same variance
//decomposition.ChopSingularValues(1E-5);
// recalculate the parameters with the chopped singular values
_decomposition.Backsubstitution(scaledY,_parameter);
_chiSquare = 0;
for(int i=0;i<numberOfData;i++)
{
double ypredicted=0;
for(int j=0;j<numberOfParameter;j++)
ypredicted += _parameter[j]*xbase[i,j];
double deviation = yarr[i]-ypredicted;
_predicted[i] = ypredicted;
_residual[i] = deviation;
_chiSquare += deviation*deviation;
}
_covarianceMatrix = _decomposition.GetCovariances();
//calculate the reduced prediction variance x'(X'X)^(-1)x
for(int i=0;i<numberOfData;i++)
{
double total = 0;
for(int j=0;j<numberOfParameter;j++)
{
double sum=0;
for(int k=0;k<numberOfParameter;k++)
sum += _covarianceMatrix[j][k]*u[i,k];
total += u[i,j]*sum;
}
_reducedPredictionVariance[i] = total;
}
return this;
}
public virtual void CalculatePredictedAndResidual(
DataTable table,
int whichY,
int numberOfFactors,
bool saveYPredicted,
bool saveYResidual,
bool saveXResidual)
{
var plsMemo = GetContentAsMultivariateContentMemento(table);
if (plsMemo == null)
throw new ArgumentException("Table does not contain a PLSContentMemento");
IMultivariateCalibrationModel calib = GetCalibrationModel(table);
// Export(table,out calib);
IMatrix matrixX = GetRawSpectra(plsMemo);
MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(matrixX.Rows, calib.NumberOfY);
MatrixMath.BEMatrix spectralResiduals = new MatrixMath.BEMatrix(matrixX.Rows, 1);
CalculatePredictedY(calib, plsMemo.SpectralPreprocessing, matrixX, numberOfFactors, predictedY, spectralResiduals);
if (saveYPredicted)
{
// insert a column with the proper name into the table and fill it
string ycolname = GetYPredicted_ColumnName(whichY, numberOfFactors);
Altaxo.Data.DoubleColumn ycolumn = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < predictedY.Rows; i++)
ycolumn[i] = predictedY[i, whichY];
table.DataColumns.Add(ycolumn, ycolname, Altaxo.Data.ColumnKind.V, GetYPredicted_ColumnGroup());
}
// subract the original y data
IMatrix matrixY = GetOriginalY(plsMemo);
MatrixMath.SubtractColumn(predictedY, matrixY, whichY, predictedY);
if (saveYResidual)
{
// insert a column with the proper name into the table and fill it
string ycolname = GetYResidual_ColumnName(whichY, numberOfFactors);
Altaxo.Data.DoubleColumn ycolumn = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < predictedY.Rows; i++)
ycolumn[i] = predictedY[i, whichY];
table.DataColumns.Add(ycolumn, ycolname, Altaxo.Data.ColumnKind.V, GetYResidual_ColumnGroup());
}
if (saveXResidual)
{
// insert a column with the proper name into the table and fill it
string ycolname = GetXResidual_ColumnName(whichY, numberOfFactors);
Altaxo.Data.DoubleColumn ycolumn = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < matrixX.Rows; i++)
{
ycolumn[i] = spectralResiduals[i, 0];
}
table.DataColumns.Add(ycolumn, ycolname, Altaxo.Data.ColumnKind.V, GetXResidual_ColumnGroup());
}
}
/// <summary>
/// Calculates the spectral leverage from preprocessed spectra.
/// </summary>
/// <param name="matrixX">Matrix of spectra (a spectrum = a row in the matrix).</param>
/// <param name="numFactors">Number of factors used for calculation.</param>
/// <returns>Matrix of spectral leverages. Normally, this is a (NumberOfPoints,1) matrix, with exception of PLS1, where it is a (NumberOfPoints,NumberOfY) matrix.</returns>
public virtual IROMatrix GetXLeverageFromPreprocessed(IROMatrix matrixX, int numFactors)
{
IMatrix result = new MatrixMath.BEMatrix(matrixX.Rows,1);
this.InternalGetXLeverageFromPreprocessed(matrixX,numFactors,result);
return result;
}
public static void Predict(
IROMatrix matrixX,
IROMatrix xLoads,
IROMatrix yLoads,
IROMatrix xScores,
IROVector crossProduct,
int numberOfFactors,
IMatrix predictedY,
IMatrix spectralResiduals)
{
int numX = xLoads.Columns;
int numY = yLoads.Columns;
int numM = yLoads.Rows;
MatrixMath.BEMatrix predictionScores = new MatrixMath.BEMatrix(numX,numY);
GetPredictionScoreMatrix(xLoads,yLoads,xScores,crossProduct,numberOfFactors,predictionScores);
MatrixMath.Multiply(matrixX,predictionScores,predictedY);
if(null!=spectralResiduals)
GetSpectralResiduals(matrixX,xLoads,yLoads,xScores,crossProduct,numberOfFactors,spectralResiduals);
}
