/// <summary>
/// This predicts the selected columns/rows against a user choosen calibration model.
/// The orientation of spectra is given by the parameter <c>spectrumIsRow</c>.
/// </summary>
/// <param name="srctable">Table holding the specta to predict values for.</param>
/// <param name="selectedColumns">Columns selected in the source table.</param>
/// <param name="selectedRows">Rows selected in the source table.</param>
/// <param name="destTable">The table to store the prediction result.</param>
/// <param name="modelTable">The table where the calibration model is stored.</param>
/// <param name="numberOfFactors">Number of factors used to predict the values.</param>
/// <param name="spectrumIsRow">If true, the spectra is horizontally oriented, else it is vertically oriented.</param>
public virtual void PredictValues(
DataTable srctable,
IAscendingIntegerCollection selectedColumns,
IAscendingIntegerCollection selectedRows,
bool spectrumIsRow,
int numberOfFactors,
DataTable modelTable,
DataTable destTable)
{
IMultivariateCalibrationModel calibModel = GetCalibrationModel(modelTable);
// Export(modelTable, out calibModel);
var memento = GetContentAsMultivariateContentMemento(modelTable);
// Fill matrixX with spectra
Altaxo.Collections.AscendingIntegerCollection spectralIndices;
Altaxo.Collections.AscendingIntegerCollection measurementIndices;
spectralIndices = new Altaxo.Collections.AscendingIntegerCollection(selectedColumns);
measurementIndices = new Altaxo.Collections.AscendingIntegerCollection(selectedRows);
RemoveNonNumericCells(srctable, measurementIndices, spectralIndices);
// exchange selection if spectrum is column
if (!spectrumIsRow)
{
Altaxo.Collections.AscendingIntegerCollection hlp;
hlp = spectralIndices;
spectralIndices = measurementIndices;
measurementIndices = hlp;
}
// if there are more data than expected, we have to map the spectral indices
if (spectralIndices.Count > calibModel.NumberOfX)
{
double[] xofx = GetXOfSpectra(srctable, spectrumIsRow, spectralIndices, measurementIndices);
string errormsg;
AscendingIntegerCollection map = MapSpectralX(calibModel.PreprocessingModel.XOfX, VectorMath.ToROVector(xofx), out errormsg);
if (map == null)
throw new ApplicationException("Can not map spectral data: " + errormsg);
else
{
AscendingIntegerCollection newspectralindices = new AscendingIntegerCollection();
for (int i = 0; i < map.Count; i++)
newspectralindices.Add(spectralIndices[map[i]]);
spectralIndices = newspectralindices;
}
}
IMatrix matrixX = GetRawSpectra(srctable, spectrumIsRow, spectralIndices, measurementIndices);
MatrixMath.BEMatrix predictedY = new MatrixMath.BEMatrix(measurementIndices.Count, calibModel.NumberOfY);
CalculatePredictedY(calibModel, memento.SpectralPreprocessing, matrixX, numberOfFactors, predictedY, null);
// now save the predicted y in the destination table
Altaxo.Data.DoubleColumn labelCol = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < measurementIndices.Count; i++)
{
labelCol[i] = measurementIndices[i];
}
destTable.DataColumns.Add(labelCol, "MeasurementLabel", Altaxo.Data.ColumnKind.Label, 0);
for (int k = 0; k < predictedY.Columns; k++)
{
Altaxo.Data.DoubleColumn predictedYcol = new Altaxo.Data.DoubleColumn();
for (int i = 0; i < measurementIndices.Count; i++)
{
predictedYcol[i] = predictedY[i, k];
}
destTable.DataColumns.Add(predictedYcol, "Predicted Y" + k.ToString(), Altaxo.Data.ColumnKind.V, 0);
}
}
/// <summary>
/// Get the matrix of x and y values (raw data).
/// </summary>
/// <param name="srctable">The table where the data come from.</param>
/// <param name="selectedColumns">The selected columns.</param>
/// <param name="selectedRows">The selected rows.</param>
/// <param name="selectedPropertyColumns">The selected property column(s).</param>
/// <param name="bHorizontalOrientedSpectrum">True if a spectrum is a single row, False if a spectrum is a single column.</param>
/// <param name="matrixX">On return, gives the matrix of spectra (each spectra is a row in the matrix).</param>
/// <param name="matrixY">On return, gives the matrix of y-values (each measurement is a row in the matrix).</param>
/// <param name="plsContent">Holds information about the analysis results.</param>
/// <param name="xOfX">On return, this is the vector of values corresponding to each spectral bin, i.e. wavelength values, frequencies etc.</param>
/// <returns></returns>
public static string GetXYMatrices(
Altaxo.Data.DataTable srctable,
IAscendingIntegerCollection selectedColumns,
IAscendingIntegerCollection selectedRows,
IAscendingIntegerCollection selectedPropertyColumns,
bool bHorizontalOrientedSpectrum,
MultivariateContentMemento plsContent,
out IMatrix matrixX,
out IMatrix matrixY,
out IROVector xOfX
)
{
matrixX = null;
matrixY = null;
xOfX = null;
plsContent.SpectrumIsRow = bHorizontalOrientedSpectrum;
Altaxo.Data.DataColumn xColumnOfX = null;
Altaxo.Data.DataColumn labelColumnOfX = new Altaxo.Data.DoubleColumn();
Altaxo.Data.DataColumnCollection concentration = bHorizontalOrientedSpectrum ? srctable.DataColumns : srctable.PropertyColumns;
// we presume for now that the spectrum is horizontally,
// if not we exchange the collections later
AscendingIntegerCollection numericDataCols = new AscendingIntegerCollection();
AscendingIntegerCollection numericDataRows = new AscendingIntegerCollection();
AscendingIntegerCollection concentrationIndices = new AscendingIntegerCollection();
AscendingIntegerCollection spectralIndices = bHorizontalOrientedSpectrum ? numericDataCols : numericDataRows;
AscendingIntegerCollection measurementIndices = bHorizontalOrientedSpectrum ? numericDataRows : numericDataCols;
plsContent.ConcentrationIndices = concentrationIndices;
plsContent.MeasurementIndices = measurementIndices;
plsContent.SpectralIndices = spectralIndices;
plsContent.SpectrumIsRow = bHorizontalOrientedSpectrum;
plsContent.OriginalDataTableName = srctable.Name;
bool bUseSelectedColumns = (null != selectedColumns && 0 != selectedColumns.Count);
// this is the number of columns (for now), but it can be less than this in case
// not all columns are numeric
int prenumcols = bUseSelectedColumns ? selectedColumns.Count : srctable.DataColumns.ColumnCount;
// check for the number of numeric columns
int numcols = 0;
for (int i = 0; i < prenumcols; i++)
{
int idx = bUseSelectedColumns ? selectedColumns[i] : i;
if (srctable[idx] is Altaxo.Data.INumericColumn)
{
numericDataCols.Add(idx);
numcols++;
}
}
// check the number of rows
bool bUseSelectedRows = (null != selectedRows && 0 != selectedRows.Count);
int numrows;
if (bUseSelectedRows)
{
numrows = selectedRows.Count;
numericDataRows.Add(selectedRows);
}
else
{
numrows = 0;
for (int i = 0; i < numcols; i++)
{
int idx = bUseSelectedColumns ? selectedColumns[i] : i;
numrows = Math.Max(numrows, srctable[idx].Count);
}
numericDataRows.Add(ContiguousIntegerRange.FromStartAndCount(0, numrows));
}
if (bHorizontalOrientedSpectrum)
{
if (numcols < 2)
return "At least two numeric columns are neccessary to do Partial Least Squares (PLS) analysis!";
// check that the selected columns are in exactly two groups
// the group which has more columns is then considered to have
// the spectrum, the other group is the y-values
int group0 = -1;
int group1 = -1;
int groupcount0 = 0;
int groupcount1 = 0;
for (int i = 0; i < numcols; i++)
{
int grp = srctable.DataColumns.GetColumnGroup(numericDataCols[i]);
if (group0 < 0)
{
group0 = grp;
groupcount0 = 1;
}
else if (group0 == grp)
{
groupcount0++;
}
else if (group1 < 0)
{
group1 = grp;
groupcount1 = 1;
}
else if (group1 == grp)
{
groupcount1++;
}
else
{
return "The columns you selected must be members of two groups (y-values and spectrum), but actually there are more than two groups!";
}
} // end for all columns
if (groupcount1 <= 0)
return "The columns you selected must be members of two groups (y-values and spectrum), but actually only one group was detected!";
if (groupcount1 < groupcount0)
{
int hlp;
hlp = groupcount1;
groupcount1 = groupcount0;
groupcount0 = hlp;
hlp = group1;
group1 = group0;
group0 = hlp;
}
// group0 is now the group of y-values (concentrations)
// group1 is now the group of x-values (spectra)
// we delete group0 from numericDataCols and add it to concentrationIndices
for (int i = numcols - 1; i >= 0; i--)
{
int index = numericDataCols[i];
if (group0 == srctable.DataColumns.GetColumnGroup(index))
{
numericDataCols.Remove(index);
concentrationIndices.Add(index);
}
}
// fill the corresponding X-Column of the spectra
xColumnOfX = Altaxo.Data.DataColumn.CreateColumnOfSelectedRows(
srctable.PropertyColumns.FindXColumnOfGroup(group1),
spectralIndices);
}
else // vertically oriented spectrum -> one spectrum is one data column
{
// we have to exchange measurementIndices and
// if PLS on columns, than we should have property columns selected
// that designates the y-values
// so count all property columns
bool bUseSelectedPropCols = (null != selectedPropertyColumns && 0 != selectedPropertyColumns.Count);
// this is the number of property columns (for now), but it can be less than this in case
// not all columns are numeric
int prenumpropcols = bUseSelectedPropCols ? selectedPropertyColumns.Count : srctable.PropCols.ColumnCount;
// check for the number of numeric property columns
for (int i = 0; i < prenumpropcols; i++)
{
int idx = bUseSelectedPropCols ? selectedPropertyColumns[i] : i;
if (srctable.PropCols[idx] is Altaxo.Data.INumericColumn)
{
concentrationIndices.Add(idx);
}
}
if (concentrationIndices.Count < 1)
return "At least one numeric property column must exist to hold the y-values!";
// fill the corresponding X-Column of the spectra
xColumnOfX = Altaxo.Data.DataColumn.CreateColumnOfSelectedRows(
srctable.DataColumns.FindXColumnOf(srctable[measurementIndices[0]]), spectralIndices);
} // else vertically oriented spectrum
IVector xOfXRW = VectorMath.CreateExtensibleVector(xColumnOfX.Count);
xOfX = xOfXRW;
if (xColumnOfX is INumericColumn)
{
for (int i = 0; i < xOfX.Length; i++)
xOfXRW[i] = ((INumericColumn)xColumnOfX)[i];
}
else
{
for (int i = 0; i < xOfX.Length; i++)
xOfXRW[i] = spectralIndices[i];
}
// now fill the matrix
// fill in the y-values
matrixY = new MatrixMath.BEMatrix(measurementIndices.Count, concentrationIndices.Count);
for (int i = 0; i < concentrationIndices.Count; i++)
{
Altaxo.Data.INumericColumn col = concentration[concentrationIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < measurementIndices.Count; j++)
{
matrixY[j, i] = col[measurementIndices[j]];
}
} // end fill in yvalues
matrixX = new MatrixMath.BEMatrix(measurementIndices.Count, spectralIndices.Count);
if (bHorizontalOrientedSpectrum)
{
for (int i = 0; i < spectralIndices.Count; i++)
{
labelColumnOfX[i] = spectralIndices[i];
Altaxo.Data.INumericColumn col = srctable[spectralIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < measurementIndices.Count; j++)
{
matrixX[j, i] = col[measurementIndices[j]];
}
} // end fill in x-values
}
else // vertical oriented spectrum
{
for (int i = 0; i < spectralIndices.Count; i++)
{
labelColumnOfX[i] = spectralIndices[i];
}
for (int i = 0; i < measurementIndices.Count; i++)
{
Altaxo.Data.INumericColumn col = srctable[measurementIndices[i]] as Altaxo.Data.INumericColumn;
for (int j = 0; j < spectralIndices.Count; j++)
{
matrixX[i, j] = col[spectralIndices[j]];
}
} // end fill in x-values
}
return null;
}
/// <summary>
/// Exports a table to a PLS2CalibrationSet
/// </summary>
/// <param name="table">The table where the calibration model is stored.</param>
/// <param name="calibrationSet"></param>
public static void Export(
DataTable table,
out PCRCalibrationModel calibrationSet)
{
int numberOfX = GetNumberOfX(table);
int numberOfY = GetNumberOfY(table);
int numberOfFactors = GetNumberOfFactors(table);
int numberOfMeasurements = GetNumberOfMeasurements(table);
calibrationSet = new PCRCalibrationModel();
calibrationSet.NumberOfX = numberOfX;
calibrationSet.NumberOfY = numberOfY;
calibrationSet.NumberOfFactors = numberOfFactors;
MultivariatePreprocessingModel preprocessSet = new MultivariatePreprocessingModel();
MultivariateContentMemento plsMemo = table.GetTableProperty("Content") as MultivariateContentMemento;
if(plsMemo!=null)
preprocessSet.PreprocessOptions = plsMemo.SpectralPreprocessing;
calibrationSet.SetPreprocessingModel(preprocessSet);
Altaxo.Collections.AscendingIntegerCollection sel = new Altaxo.Collections.AscendingIntegerCollection();
Altaxo.Data.DataColumn col;
col = table[GetXOfX_ColumnName()];
if(col==null || !(col is INumericColumn)) NotFound(GetXOfX_ColumnName());
preprocessSet.XOfX = Altaxo.Calc.LinearAlgebra.DataColumnWrapper.ToROVector((INumericColumn)col,numberOfX);
col = table[GetXMean_ColumnName()];
if(col==null) NotFound(GetXMean_ColumnName());
preprocessSet.XMean = Altaxo.Calc.LinearAlgebra.DataColumnWrapper.ToROVector(col,numberOfX);
col = table[GetXScale_ColumnName()];
if(col==null) NotFound(GetXScale_ColumnName());
preprocessSet.XScale = Altaxo.Calc.LinearAlgebra.DataColumnWrapper.ToROVector(col,numberOfX);
sel.Clear();
col = table[GetYMean_ColumnName()];
if(col==null) NotFound(GetYMean_ColumnName());
sel.Add(table.DataColumns.GetColumnNumber(col));
preprocessSet.YMean = DataColumnWrapper.ToROVector(col,numberOfY);
sel.Clear();
col = table[GetYScale_ColumnName()];
if(col==null) NotFound(GetYScale_ColumnName());
sel.Add(table.DataColumns.GetColumnNumber(col));
preprocessSet.YScale = DataColumnWrapper.ToROVector(col,numberOfY);
sel.Clear();
for(int i=0;i<numberOfFactors;i++)
{
string colname = GetXScore_ColumnName(i);
col = table[colname];
if(col==null) NotFound(colname);
sel.Add(table.DataColumns.GetColumnNumber(col));
}
calibrationSet.XScores = DataTableWrapper.ToROColumnMatrix(table.DataColumns,sel,numberOfMeasurements);
sel.Clear();
for(int i=0;i<numberOfFactors;i++)
{
string colname = GetXLoad_ColumnName(i);
col = table[colname];
if(col==null) NotFound(colname);
sel.Add(table.DataColumns.GetColumnNumber(col));
}
calibrationSet.XLoads = DataTableWrapper.ToRORowMatrix(table.DataColumns,sel,numberOfX);
sel.Clear();
for(int i=0;i<numberOfY;i++)
{
string colname = GetYLoad_ColumnName(i);
col = table[colname];
if(col==null) NotFound(colname);
sel.Add(table.DataColumns.GetColumnNumber(col));
}
calibrationSet.YLoads = DataTableWrapper.ToROColumnMatrix(table.DataColumns,sel,numberOfMeasurements);
sel.Clear();
col = table[GetCrossProduct_ColumnName()];
if(col==null) NotFound(GetCrossProduct_ColumnName());
calibrationSet.CrossProduct = Altaxo.Calc.LinearAlgebra.DataColumnWrapper.ToROVector(col,numberOfFactors);
}
public static LinearFitBySvd Regress(MultivariateLinearFitParameters parameters, out string[] paramNames)
{
DataColumnCollection table = parameters.Table;
IAscendingIntegerCollection selectedCols = parameters.SelectedDataColumns;
AscendingIntegerCollection selectedColsWODependent = new AscendingIntegerCollection(selectedCols);
selectedColsWODependent.RemoveAt(parameters.DependentColumnIndexIntoSelection);
IAscendingIntegerCollection validRows = DataTableWrapper.GetCollectionOfValidNumericRows(parameters.Table,selectedCols);
parameters.SelectedDataRows = validRows;
IROMatrix xbase;
if(parameters.IncludeIntercept)
{
xbase = DataTableWrapper.ToROColumnMatrixWithIntercept(parameters.Table,selectedColsWODependent,validRows);
}
else
{
xbase = DataTableWrapper.ToROColumnMatrix(parameters.Table,selectedColsWODependent,validRows);
}
paramNames = new string[xbase.Columns];
if(parameters.IncludeIntercept)
{
paramNames[0] = "Intercept";
for(int i=0;i<selectedColsWODependent.Count;i++)
paramNames[i+1]=table[selectedColsWODependent[i]].Name;
}
else
{
for(int i=0;i<selectedColsWODependent.Count;i++)
paramNames[i]=table[selectedColsWODependent[i]].Name;
}
// Fill the y and the error array
double[] yarr = new double[validRows.Count];
double[] earr = new double[validRows.Count];
Altaxo.Data.INumericColumn ycol = (Altaxo.Data.INumericColumn)table[selectedCols[parameters.DependentColumnIndexIntoSelection]];
for(int i=0;i<validRows.Count;i++)
{
yarr[i] = ycol[validRows[i]];
earr[i] = 1;
}
LinearFitBySvd fit =
new LinearFitBySvd(
xbase,yarr,earr, xbase.Rows, xbase.Columns, 1E-5);
return fit;
}
/// <summary>
/// Calculates the valid numeric rows of the data source, i.e. that rows that can be used for fitting. Both dependent and independent variable sources are considered. A row is considered valid, if
/// all (independent and dependent) variables of this row have finite numeric values.
/// </summary>
/// <returns>The set of rows that can be used for fitting.</returns>
public IAscendingIntegerCollection CalculateValidNumericRows()
{
// also obtain the valid rows both of the independent and of the dependent variables
var cols = new IReadableColumn[_independentVariables.Length + _dependentVariables.Length];
int i;
AscendingIntegerCollection selectedCols = new AscendingIntegerCollection();
// note: for a fitting session all independent variables columns must
// be not null
int maxLength = int.MaxValue;
for (i = 0; i < _independentVariables.Length; i++)
{
cols[i] = _independentVariables[i].Document;
selectedCols.Add(i);
if (cols[i].Count.HasValue)
maxLength = Math.Min(maxLength, cols[i].Count.Value);
}
// note: for a fitting session some of the dependent variables can be null
for (int j = 0; j < _dependentVariables.Length; ++j, ++i)
{
if (_dependentVariables[j] != null && _dependentVariables[j].Document != null)
{
cols[i] = _dependentVariables[j].Document;
selectedCols.Add(i);
if (cols[i].Count.HasValue)
maxLength = Math.Min(maxLength, cols[i].Count.Value);
}
}
if (maxLength == int.MaxValue)
maxLength = 0; // in case non of the columns has a defined length
bool[] arr = Altaxo.Calc.LinearAlgebra.DataTableWrapper.GetValidNumericRows(cols, selectedCols, _rangeOfRows.GetSelectedRowIndicesFromTo(0, maxLength, _dataTable?.Document.DataColumns, maxLength), maxLength);
return Altaxo.Calc.LinearAlgebra.DataTableWrapper.GetCollectionOfValidNumericRows(arr);
}
public IAscendingIntegerCollection CalculateValidNumericRows()
{
// also obtain the valid rows both of the independent and of the dependent variables
INumericColumn[] cols = new INumericColumn[_independentVariables.Length + _dependentVariables.Length];
int i;
AscendingIntegerCollection selectedCols = new AscendingIntegerCollection();
// note: for a fitting session all independent variables columns must
// be not null
int maxLength = int.MaxValue;
for (i = 0; i < _independentVariables.Length; i++)
{
cols[i] = _independentVariables[i].Document;
selectedCols.Add(i);
if (cols[i] is IDefinedCount)
maxLength = Math.Min(maxLength, ((IDefinedCount)cols[i]).Count);
}
// note: for a fitting session some of the dependent variables can be null
for (int j = 0; j < _dependentVariables.Length; ++j, ++i)
{
if (_dependentVariables[j] != null && _dependentVariables[j].Document != null)
{
cols[i] = _dependentVariables[j].Document;
selectedCols.Add(i);
if (cols[i] is IDefinedCount)
maxLength = Math.Min(maxLength, ((IDefinedCount)cols[i]).Count);
}
}
if (maxLength == int.MaxValue)
maxLength = 0;
maxLength = Math.Min(maxLength, this._rangeOfRows.End);
bool[] arr = Altaxo.Calc.LinearAlgebra.DataTableWrapper.GetValidNumericRows(cols, selectedCols, maxLength);
return Altaxo.Calc.LinearAlgebra.DataTableWrapper.GetCollectionOfValidNumericRows(arr);
}
/// <summary>
/// Writes ASCII to the clipboard if data cells are selected.
/// </summary>
/// <param name="dt">The data table.</param>
/// <param name="selCols">Selected data columns.</param>
/// <param name="selRows">Selected data rows.</param>
/// <param name="selPropCols">Selected property columns.</param>
/// <param name="dao">The clipboard data object</param>
public static void WriteAsciiToClipBoardIfDataCellsSelected(
DataTable dt,
Altaxo.Collections.AscendingIntegerCollection selCols,
Altaxo.Collections.AscendingIntegerCollection selRows,
Altaxo.Collections.AscendingIntegerCollection selPropCols,
Altaxo.Gui.IClipboardSetDataObject dao)
{
if (selCols.Count == 0)
{
selCols = new Altaxo.Collections.AscendingIntegerCollection();
selCols.AddRange(0, dt.DataColumnCount);
}
if (selRows.Count == 0)
{
selRows = new Altaxo.Collections.AscendingIntegerCollection();
int nRows = 0; // count the rows since they are maybe less than in the hole worksheet
for (int i = 0; i < selCols.Count; i++)
{
nRows = System.Math.Max(nRows, dt[selCols[i]].Count);
}
selRows.AddRange(0, nRows);
}
System.IO.StringWriter str = new System.IO.StringWriter();
for (int i = 0; i < selPropCols.Count; i++)
{
for (int j = 0; j < selCols.Count; j++)
{
str.Write(dt.PropertyColumns[selPropCols[i]][selCols[j]].ToString());
if (j < selCols.Count - 1)
str.Write(";");
else
str.WriteLine();
}
}
for (int i = 0; i < selRows.Count; i++)
{
for (int j = 0; j < selCols.Count; j++)
{
str.Write(dt.DataColumns[selCols[j]][selRows[i]].ToString());
if (j < selCols.Count - 1)
str.Write(";");
else
str.WriteLine();
}
}
dao.SetCommaSeparatedValues(str.ToString());
// now also as tab separated text
System.IO.StringWriter sw = new System.IO.StringWriter();
for (int i = 0; i < selPropCols.Count; i++)
{
for (int j = 0; j < selCols.Count; j++)
{
str.Write(dt.PropertyColumns[selPropCols[i]][selCols[j]].ToString());
if (j < selCols.Count - 1)
str.Write("\t");
else
str.WriteLine();
}
}
for (int i = 0; i < selRows.Count; i++)
{
for (int j = 0; j < selCols.Count; j++)
{
sw.Write(dt.DataColumns[selCols[j]][selRows[i]].ToString());
if (j < selCols.Count - 1)
sw.Write("\t");
else
sw.WriteLine();
}
}
dao.SetData(typeof(string), sw.ToString());
}
/// <summary>
/// Expand the source columns according to the provided options. The source table and the settings are provided in the <paramref name="options"/> variable.
/// The provided destination table is cleared from all data and property values before.
/// </summary>
/// <param name="inputData">The data containing the source table, the participating columns and the column with the cycling variable.</param>
/// <param name="options">The settings for expanding.</param>
/// <param name="destTable">The destination table. Any data will be removed before filling with the new data.</param>
/// <returns>Null if the method finishes successfully, or an error information.</returns>
public static string ExpandCyclingVariableColumn(DataTableMultipleColumnProxy inputData, ExpandCyclingVariableColumnOptions options, DataTable destTable)
{
var srcTable = inputData.DataTable;
try
{
ExpandCyclingVariableColumnDataAndOptions.EnsureCoherence(inputData, true);
}
catch (Exception ex)
{
return ex.Message;
}
destTable.DataColumns.RemoveColumnsAll();
destTable.PropCols.RemoveColumnsAll();
DataColumn srcCycCol = inputData.GetDataColumnOrNull(ExpandCyclingVariableColumnDataAndOptions.ColumnWithCyclingVariableIdentifier);
var repeatRanges = DecomposeIntoRepeatUnits(srcCycCol);
// check if there is at least one averaged column
var columnsToAverageOverRepeatPeriod = inputData.GetDataColumns(ExpandCyclingVariableColumnDataAndOptions.ColumnsToAverageIdentifier);
if (options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.FirstAveragedColumn && columnsToAverageOverRepeatPeriod.Count == 0)
throw new ArgumentException("In order to let the first averaged column being the x-column, a column to average is needed, but the options didn't provide such column!");
// get the other columns to process
var srcColumnsToProcess = new List<DataColumn>(inputData.GetDataColumns(ExpandCyclingVariableColumnDataAndOptions.ColumnsParticipatingIdentifier));
// subtract cyclic variable column and average columns
srcColumnsToProcess.Remove(srcCycCol);
foreach (var col in columnsToAverageOverRepeatPeriod)
srcColumnsToProcess.Remove(col);
// --- Create and calculate the averaged columns, for now only temporarily ---
var propColsTemp = AverageColumns(srcTable, columnsToAverageOverRepeatPeriod, options, repeatRanges);
// --- avgValueOrder designates the ordering of the first averaged column and therefore of the sorting of the ranges and of the first averaged column
int[] avgValueOrder = Sorting.CreateIdentityIndices(repeatRanges.Count);
// --- prepare the sorting of columns by first averaged column ---
var rangeOrderSorting = options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.CyclingVariable ? options.DestinationColumnSorting : options.DestinationRowSorting;
if (propColsTemp.Length > 0 && rangeOrderSorting != ExpandCyclingVariableColumnOptions.OutputSorting.None)
{
avgValueOrder = Sorting.HeapSortVirtually(propColsTemp[0], avgValueOrder);
if (rangeOrderSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending)
Sorting.ReverseArray(avgValueOrder);
}
// prepare the sorting of the cycling values
var cycValueSorting = options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.CyclingVariable ? options.DestinationRowSorting : options.DestinationColumnSorting;
// create a dictionary with the cycling values (unique) and the corresponding ordering index
var cycValueOrder = GetUniqueValues(srcCycCol, cycValueSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending);
if (options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.CyclingVariable)
{
int[] propColsIdx = CreatePropColsForAveragedColumns(srcTable, columnsToAverageOverRepeatPeriod, options, destTable);
// --- Fill the x column, take the row sorting into account ---
var destXCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCycCol), srcCycCol.GetType(), ColumnKind.X, srcTable.DataColumns.GetColumnGroup(srcCycCol));
foreach (var entry in cycValueOrder)
destXCol[entry.Value] = entry.Key;
if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupOneColumn)
{
// foreach sourceColumnToProcess create as many destination columns as there are cycling ranges available
foreach (var srcCol in srcColumnsToProcess)
{
int nCreatedCol = -1;
var destColumnsToSort = new AscendingIntegerCollection();
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nCreatedCol;
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCreatedCol.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
destColumnsToSort.Add(nDestCol);
foreach (var nSrcRow in range)
{
int nDestRow = cycValueOrder[srcCycCol[nSrcRow]];
destCol[nDestRow] = srcCol[nSrcRow];
}
// fill also property columns
for (int nPropCol = 0; nPropCol < propColsTemp.Length; nPropCol++)
{
destTable.PropCols[propColsIdx[nPropCol]][nDestCol] = propColsTemp[nPropCol][rangeIndex];
}
}
} // repeat for each source colum to process
}
else if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupAllColumns)
{
int nCreatedCol = -1; // running number of processed range for column creation (Naming)
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nCreatedCol;
foreach (var srcCol in srcColumnsToProcess)
{
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCreatedCol.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
foreach (var nSrcRow in range)
{
int nDestRow = cycValueOrder[srcCycCol[nSrcRow]];
destCol[nDestRow] = srcCol[nSrcRow];
}
// fill also property columns
for (int nPropCol = 0; nPropCol < propColsTemp.Length; nPropCol++)
{
destTable.PropCols[propColsIdx[nPropCol]][nDestCol] = propColsTemp[nPropCol][rangeIndex];
}
}
}
}
else
{
throw new NotImplementedException("The option for destination output is unknown: " + options.DestinationOutput.ToString());
}
}
else if (options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.FirstAveragedColumn)
{
// now the first x column contains the values of the averaged column
// the rest of the data columns is repeated as many times as there are members in each repeat range
DataColumn srcXCol = columnsToAverageOverRepeatPeriod[0];
var destXCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcXCol), srcXCol.GetType(), ColumnKind.X, srcTable.DataColumns.GetColumnGroup(srcXCol));
// Fill with destination X
for (int nDestRow = 0; nDestRow < repeatRanges.Count; nDestRow++)
destXCol[nDestRow] = propColsTemp[0][avgValueOrder[nDestRow]];
// the only property column that is now usefull is that with the repeated values
var destPropCol = destTable.PropCols.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCycCol), srcCycCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCycCol), srcTable.DataColumns.GetColumnGroup(srcCycCol));
if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupOneColumn)
{
foreach (var srcCol in srcColumnsToProcess)
{
int nCurrNumber = -1;
IEnumerable<AltaxoVariant> cycValues = cycValueOrder.Keys;
if (options.DestinationColumnSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending)
cycValues = cycValueOrder.Keys.Reverse();
foreach (var cycValue in cycValues)
{
++nCurrNumber;
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCurrNumber.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
int nDestRow = -1;
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nDestRow;
int nSrcRow = FindSrcXRow(srcCycCol, cycValue, range);
if (nSrcRow >= 0)
destCol[nDestRow] = srcCol[nSrcRow];
}
// fill also property column
destPropCol[nDestCol] = cycValue;
}
}
}
else if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupAllColumns)
{
IEnumerable<AltaxoVariant> positionsKeys = cycValueOrder.Keys;
if (options.DestinationColumnSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending)
positionsKeys = cycValueOrder.Keys.Reverse();
int nCurrNumber = -1;
foreach (var xVal in positionsKeys)
{
++nCurrNumber;
foreach (var srcCol in srcColumnsToProcess)
{
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCurrNumber.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
int nDestRow = -1;
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nDestRow;
int nSrcRow = FindSrcXRow(srcCycCol, xVal, range);
if (nSrcRow >= 0)
destCol[nDestRow] = srcCol[nSrcRow];
}
// fill also property column
destPropCol[nDestCol] = xVal;
}
}
}
else
{
throw new NotImplementedException("The option for destination output is unknown: " + options.DestinationOutput.ToString());
}
}
return null;
}
/// <summary>
/// Gets the indices of data columns that can participate in a matrix area, by providing a data table and the selected column.
/// The participating data columns must have ColumnKind.V, and must share the same group number.
/// </summary>
/// <param name="table">The table.</param>
/// <param name="selectedColumns">The selected data columns of the provided table. You can provide <c>null</c> for this parameter. This is considered as if all columns of the table are selected.</param>
/// <returns></returns>
public static AscendingIntegerCollection GetParticipatingDataColumns(DataTable table, IAscendingIntegerCollection selectedColumns)
{
var result = new Altaxo.Collections.AscendingIntegerCollection();
int? groupNumber = null;
if (selectedColumns == null || selectedColumns.Count == 0) // No columns selected - than we assume all columns, but only V-columns
{
var dc = table.DataColumns;
for (int i = 0; i < table.DataColumnCount; ++i)
{
if (dc[i] is Altaxo.Data.INumericColumn && dc.GetColumnKind(i) == ColumnKind.V && (!groupNumber.HasValue || groupNumber.Value == dc.GetColumnGroup(i)))
{
result.Add(i);
}
}
}
else
{
for (int k = 0; k < selectedColumns.Count; ++k)
{
var dc = table.DataColumns;
int i = selectedColumns[k];
if (dc[i] is DoubleColumn && dc.GetColumnKind(i) == ColumnKind.V && (!groupNumber.HasValue || groupNumber.Value == dc.GetColumnGroup(i)))
{
result.Add(i);
}
}
}
return result;
}
/// <summary>
/// Gets the data rows that participate in a matrix area by providing a table, the collection of selected data rows, and the collection of selected data columns.
/// </summary>
/// <param name="table">The table.</param>
/// <param name="selectedRows">The selected data rows.</param>
/// <param name="participatingColumns">The data columns that participate in the matrix area.</param>
/// <returns>The collection of indices of data rows that participate in the matrix area.</returns>
public static Altaxo.Collections.AscendingIntegerCollection GetParticipatingDataRows(DataTable table, IAscendingIntegerCollection selectedRows, IAscendingIntegerCollection participatingColumns)
{
var result = new AscendingIntegerCollection();
if (null != selectedRows && selectedRows.Count > 0)
{
result.Add(selectedRows);
}
else
{
var dc = table.DataColumns;
int rows = participatingColumns.Select(i => dc[i].Count).MinOrDefault(0);
result.AddRange(0, rows);
}
return result;
}
/// <summary>
/// Creates a collection cloned from another <see cref="AscendingIntegerCollection" />.
/// </summary>
/// <param name="from"></param>
public AscendingIntegerCollection(AscendingIntegerCollection from)
{
_list = new System.Collections.Generic.SortedList<int, object>(from._list);
}
/// <summary>
/// Initializes a new instance of the <see cref="DataTableMultipleColumnProxy"/> class. The selected collections determine which columns and rows contribute to this instance.
/// The group number is determined by the first selected column (or, if no column is selected, by the first column of the data table).
/// </summary>
/// <param name="identifier">The identifier of the bundle of columns that are initially set with this constructor.</param>
/// <param name="table">The underlying table.</param>
/// <param name="selectedDataRows">The selected data rows.</param>
/// <param name="selectedDataColumns">The selected data columns.</param>
/// <exception cref="System.ArgumentNullException">table must not be null.</exception>
public DataTableMultipleColumnProxy(string identifier, DataTable table, IAscendingIntegerCollection selectedDataRows, IAscendingIntegerCollection selectedDataColumns)
{
if (null == identifier)
throw new ArgumentNullException("identifier");
if (null == table)
throw new ArgumentNullException("table");
_dataColumnBundles = new Dictionary<string, ColumnBundleInfo>();
_dataTable = new DataTableProxy(table) { ParentObject = this };
_groupNumber = 0;
if (null != selectedDataColumns && selectedDataColumns.Count > 0)
_groupNumber = table.DataColumns.GetColumnGroup(table[selectedDataColumns[0]]);
var bundle = new ColumnBundleInfo();
_dataColumnBundles.Add(identifier, bundle);
int maxRowCount = 0;
if (selectedDataColumns != null && selectedDataColumns.Count > 0)
{
for (int i = 0; i < selectedDataColumns.Count; ++i)
{
var col = table[selectedDataColumns[i]];
if (table.DataColumns.GetColumnGroup(col) == _groupNumber)
{
InternalAddDataColumnNoClone(bundle, ReadableColumnProxyBase.FromColumn(col));
maxRowCount = Math.Max(maxRowCount, col.Count);
}
}
}
else // nothing selected - use all columns of group number 0
{
for (int i = 0; i < table.DataColumnCount; ++i)
{
var col = table[i];
if (table.DataColumns.GetColumnGroup(col) == _groupNumber)
{
InternalAddDataColumnNoClone(bundle, ReadableColumnProxyBase.FromColumn(col));
maxRowCount = Math.Max(maxRowCount, col.Count);
}
}
}
_useAllAvailableDataRows = null == selectedDataRows || selectedDataRows.Count == 0;
_participatingDataRows = new AscendingIntegerCollection(_useAllAvailableDataRows ? ContiguousIntegerRange.FromStartAndCount(0, maxRowCount) : selectedDataRows);
}
/// <summary>
/// Copies data from another instance of <see cref="DataTableMultipleColumnProxy"/>.
/// </summary>
/// <param name="obj">The instance.</param>
/// <returns><c>True</c> if any data could be copyied.</returns>
public bool CopyFrom(object obj)
{
if (object.ReferenceEquals(this, obj))
return true;
var from = obj as DataTableMultipleColumnProxy;
if (null == from)
return false;
InternalSetDataTable((DataTableProxy)from._dataTable.Clone());
InternalSetDataColumnsWithCloning(from._dataColumnBundles);
this._groupNumber = from._groupNumber;
this._useAllAvailableDataRows = from._useAllAvailableDataRows;
_participatingDataRows = (AscendingIntegerCollection)from._participatingDataRows.Clone();
_isDirty = from._isDirty;
return true;
}
/// <summary>
/// This maps the indices of a master x column to the indices of a column to map.
/// </summary>
/// <param name="xmaster">The master column containing x-values, for instance the spectral wavelength of the PLS calibration model.</param>
/// <param name="xtomap">The column to map containing x-values, for instance the spectral wavelength of an unknown spectra to predict.</param>
/// <param name="failureMessage">In case of a mapping error, contains detailed information about the error.</param>
/// <returns>The indices of the mapping column that matches those of the master column. Contains as many indices as items in xmaster. In case of mapping error, returns null.</returns>
public static Altaxo.Collections.AscendingIntegerCollection MapSpectralX(IROVector xmaster, IROVector xtomap, out string failureMessage)
{
failureMessage = null;
int mastercount = xmaster.Length;
int mapcount = xtomap.Length;
if (mapcount < mastercount)
{
failureMessage = string.Format("More items to map ({0} than available ({1}", mastercount, mapcount);
return null;
}
Altaxo.Collections.AscendingIntegerCollection result = new Altaxo.Collections.AscendingIntegerCollection();
// return an empty collection if there is nothing to map
if (mastercount == 0)
return result;
// there is only one item to map - we can not check this - return a 1:1 map
if (mastercount == 1)
{
result.Add(0);
return result;
}
// presumtion here (checked before): mastercount>=2, mapcount>=1
double distanceback, distancecurrent, distanceforward;
int i, j;
for (i = 0, j = 0; i < mastercount && j < mapcount; j++)
{
distanceback = j == 0 ? double.MaxValue : Math.Abs(xtomap[j - 1] - xmaster[i]);
distancecurrent = Math.Abs(xtomap[j] - xmaster[i]);
distanceforward = (j + 1) >= mapcount ? double.MaxValue : Math.Abs(xtomap[j + 1] - xmaster[i]);
if (distanceback < distancecurrent)
{
failureMessage = string.Format("Mapping error - distance of master[{0}] to current map[{1}] is greater than to previous map[{2}]", i, j, j - 1);
return null;
}
else if (distanceforward < distancecurrent)
continue;
else
{
result.Add(j);
i++;
}
}
if (i != mastercount)
{
failureMessage = string.Format("Mapping error- no mapping found for current master[{0}]", i - 1);
return null;
}
return result;
}
/// <summary>
/// Writes ASCII to the clipboard if only property cells are selected.
/// </summary>
/// <param name="dg">The worksheet controller</param>
/// <param name="dao">The clipboard data object</param>
protected static void WriteAsciiToClipBoardIfOnlyPropertyCellsSelected(IWorksheetController dg, Altaxo.Gui.IClipboardSetDataObject dao)
{
// columns are selected
DataTable dt = dg.DataTable;
Altaxo.Collections.AscendingIntegerCollection selCols = dg.SelectedPropertyColumns;
Altaxo.Collections.AscendingIntegerCollection selRows = dg.SelectedPropertyRows;
if (selRows.Count == 0)
{
selRows = new Altaxo.Collections.AscendingIntegerCollection();
selRows.AddRange(0, dg.DataTable.PropertyRowCount);
}
System.IO.StringWriter str = new System.IO.StringWriter();
for (int i = 0; i < selRows.Count; i++)
{
for (int j = 0; j < selCols.Count; j++)
{
str.Write(dt.PropertyColumns[selCols[j]][selRows[i]].ToString());
if (j < selCols.Count - 1)
str.Write(";");
else
str.WriteLine();
}
}
dao.SetCommaSeparatedValues(str.ToString());
// now also as tab separated text
System.IO.StringWriter sw = new System.IO.StringWriter();
for (int i = 0; i < selRows.Count; i++)
{
for (int j = 0; j < selCols.Count; j++)
{
sw.Write(dt.PropertyColumns[selCols[j]][selRows[i]].ToString());
if (j < selCols.Count - 1)
sw.Write("\t");
else
sw.WriteLine();
}
}
dao.SetData(typeof(string), sw.ToString());
}
/// <summary>
/// Plots all preprocessed spectra into a newly created graph.
/// </summary>
/// <param name="table">The table of PLS output data.</param>
public static void PlotPredictionScores(Altaxo.Data.DataTable table)
{
MultivariateContentMemento plsMemo = table.GetTableProperty("Content") as MultivariateContentMemento;
if(plsMemo==null)
return;
if(plsMemo.PreferredNumberOfFactors<=0)
QuestPreferredNumberOfFactors(plsMemo);
GetAnalysis(table).CalculateAndStorePredictionScores(table, plsMemo.PreferredNumberOfFactors);
AscendingIntegerCollection sel = new AscendingIntegerCollection();
for(int i=0;i<plsMemo.NumberOfConcentrationData;i++)
{
string name = WorksheetAnalysis.GetPredictionScore_ColumnName(i,plsMemo.PreferredNumberOfFactors);
if(null!=table[name])
sel.Add(table.DataColumns.GetColumnNumber(table[name]));
}
Worksheet.Commands.PlotCommands.PlotLine(table,sel,true,false);
}
/// <summary>
/// Gets the collection of valid rows from the array that is returned by <see cref="GetValidNumericRows(INumericColumn[], IAscendingIntegerCollection, int)" />.
/// </summary>
/// <param name="array">The boolean array.</param>
/// <returns>An collection of ascending integer values. These values are the indizes of valid numeric rows, i.e. the number of elements in the array which have the value of true.</returns>
public static AscendingIntegerCollection GetCollectionOfValidNumericRows(bool[] array)
{
AscendingIntegerCollection result = new AscendingIntegerCollection();
for (int i = 0; i < array.Length; i++)
{
if (array[i])
result.Add(i);
}
return result;
}