/// <summary>
/// Plots all preprocessed spectra into a newly created graph.
/// </summary>
/// <param name="table">The table of PLS output data.</param>
public static void PlotPredictionScores(Altaxo.Data.DataTable table)
{
MultivariateContentMemento plsMemo = table.GetTableProperty("Content") as MultivariateContentMemento;
if (plsMemo == null)
{
return;
}
if (plsMemo.PreferredNumberOfFactors <= 0)
{
QuestPreferredNumberOfFactors(plsMemo);
}
GetAnalysis(table).CalculateAndStorePredictionScores(table, plsMemo.PreferredNumberOfFactors);
AscendingIntegerCollection sel = new AscendingIntegerCollection();
for (int i = 0; i < plsMemo.NumberOfConcentrationData; i++)
{
string name = WorksheetAnalysis.GetPredictionScore_ColumnName(i, plsMemo.PreferredNumberOfFactors);
if (null != table[name])
{
sel.Add(table.DataColumns.GetColumnNumber(table[name]));
}
}
Worksheet.Commands.PlotCommands.PlotLine(table, sel, true, false);
}
public static LinearFitBySvd Regress(MultivariateLinearFitParameters parameters, out string[] paramNames)
{
DataColumnCollection table = parameters.Table;
IAscendingIntegerCollection selectedCols = parameters.SelectedDataColumns;
var selectedColsWODependent = new AscendingIntegerCollection(selectedCols);
selectedColsWODependent.RemoveAt(parameters.DependentColumnIndexIntoSelection);
IAscendingIntegerCollection validRows = DataTableWrapper.GetCollectionOfValidNumericRows(parameters.Table, selectedCols);
parameters.SelectedDataRows = validRows;
IROMatrix <double> xbase;
if (parameters.IncludeIntercept)
{
xbase = DataTableWrapper.ToROColumnMatrixWithIntercept(parameters.Table, selectedColsWODependent, validRows);
}
else
{
xbase = DataTableWrapper.ToROColumnMatrix(parameters.Table, selectedColsWODependent, validRows);
}
paramNames = new string[xbase.ColumnCount];
if (parameters.IncludeIntercept)
{
paramNames[0] = "Intercept";
for (int i = 0; i < selectedColsWODependent.Count; i++)
{
paramNames[i + 1] = table[selectedColsWODependent[i]].Name;
}
}
else
{
for (int i = 0; i < selectedColsWODependent.Count; i++)
{
paramNames[i] = table[selectedColsWODependent[i]].Name;
}
}
// Fill the y and the error array
double[] yarr = new double[validRows.Count];
double[] earr = new double[validRows.Count];
var ycol = (Altaxo.Data.INumericColumn)table[selectedCols[parameters.DependentColumnIndexIntoSelection]];
for (int i = 0; i < validRows.Count; i++)
{
yarr[i] = ycol[validRows[i]];
earr[i] = 1;
}
var fit =
new LinearFitBySvd(
xbase, yarr, earr, xbase.RowCount, xbase.ColumnCount, 1E-5);
return(fit);
}
/// <summary>
/// Gets the collection of valid rows from the array that is returned by <see cref="GetValidNumericRows(INumericColumn[], IAscendingIntegerCollection, int)" />.
/// </summary>
/// <param name="array">The boolean array.</param>
/// <returns>An collection of ascending integer values. These values are the indizes of valid numeric rows, i.e. the number of elements in the array which have the value of true.</returns>
public static AscendingIntegerCollection GetCollectionOfValidNumericRows(bool[] array)
{
AscendingIntegerCollection result = new AscendingIntegerCollection();
for (int i = 0; i < array.Length; i++)
{
if (array[i])
{
result.Add(i);
}
}
return(result);
}
/// <summary>
/// Gets the data rows that participate in a matrix area by providing a table, the collection of selected data rows, and the collection of selected data columns.
/// </summary>
/// <param name="table">The table.</param>
/// <param name="selectedRows">The selected data rows.</param>
/// <param name="participatingColumns">The data columns that participate in the matrix area.</param>
/// <returns>The collection of indices of data rows that participate in the matrix area.</returns>
public static Altaxo.Collections.AscendingIntegerCollection GetParticipatingDataRows(DataTable table, IAscendingIntegerCollection selectedRows, IAscendingIntegerCollection participatingColumns)
{
var result = new AscendingIntegerCollection();
if (null != selectedRows && selectedRows.Count > 0)
{
result.Add(selectedRows);
}
else
{
var dc = table.DataColumns;
int rows = participatingColumns.Select(i => dc[i].Count).MaxOrDefault(0);
result.AddRange(0, rows);
}
return(result);
}
public IAscendingIntegerCollection CalculateValidNumericRows()
{
// also obtain the valid rows both of the independent and of the dependent variables
INumericColumn[] cols = new INumericColumn[_independentVariables.Length + _dependentVariables.Length];
int i;
AscendingIntegerCollection selectedCols = new AscendingIntegerCollection();
// note: for a fitting session all independent variables columns must
// be not null
int maxLength = int.MaxValue;
for (i = 0; i < _independentVariables.Length; i++)
{
cols[i] = _independentVariables[i].Document;
selectedCols.Add(i);
if (cols[i] is IDefinedCount)
{
maxLength = Math.Min(maxLength, ((IDefinedCount)cols[i]).Count);
}
}
// note: for a fitting session some of the dependent variables can be null
for (int j = 0; j < _dependentVariables.Length; ++j, ++i)
{
if (_dependentVariables[j] != null && _dependentVariables[j].Document != null)
{
cols[i] = _dependentVariables[j].Document;
selectedCols.Add(i);
if (cols[i] is IDefinedCount)
{
maxLength = Math.Min(maxLength, ((IDefinedCount)cols[i]).Count);
}
}
}
if (maxLength == int.MaxValue)
{
maxLength = 0;
}
maxLength = Math.Min(maxLength, this._rangeOfRows.End);
bool[] arr = Altaxo.Calc.LinearAlgebra.DataTableWrapper.GetValidNumericRows(cols, selectedCols, maxLength);
return(Altaxo.Calc.LinearAlgebra.DataTableWrapper.GetCollectionOfValidNumericRows(arr));
}
/// <summary>
/// Copies data from another instance of <see cref="DataTableMultipleColumnProxy"/>.
/// </summary>
/// <param name="obj">The instance.</param>
/// <returns><c>True</c> if any data could be copyied.</returns>
public bool CopyFrom(object obj)
{
if (object.ReferenceEquals(this, obj))
{
return(true);
}
var from = obj as DataTableMultipleColumnProxy;
if (null == from)
{
return(false);
}
InternalSetDataTable((DataTableProxy)from._dataTable.Clone());
InternalSetDataColumnsWithCloning(from._dataColumnBundles);
_groupNumber = from._groupNumber;
_useAllAvailableDataRows = from._useAllAvailableDataRows;
_participatingDataRows = (AscendingIntegerCollection)from._participatingDataRows.Clone();
_isDirty = from._isDirty;
return(true);
}
public static List <IGPlotItem> CreatePlotItems(DataTable table, IAscendingIntegerCollection selectedColumns, G2DPlotStyleCollection templatePlotStyle)
{
int len = selectedColumns.Count;
int numColumns = table.DataColumnCount;
List <IGPlotItem> result = new List <IGPlotItem>();
ErrorBarPlotStyle unpairedPositiveError = null;
ErrorBarPlotStyle unpairedNegativeError = null;
AscendingIntegerCollection processedColumns = new AscendingIntegerCollection();
int idx;
for (int sci = 0; sci < len; sci++)
{
idx = selectedColumns[sci];
if (processedColumns.Contains(idx))
{
continue;
}
else
{
processedColumns.Add(idx);
}
Altaxo.Data.DataColumn ycol = table[idx];
Altaxo.Data.DataColumn xcol = table.DataColumns.FindXColumnOf(ycol);
XYColumnPlotData pa;
if (null != xcol)
{
pa = new XYColumnPlotData(xcol, ycol);
}
else
{
pa = new XYColumnPlotData(new Altaxo.Data.IndexerColumn(), ycol);
}
G2DPlotStyleCollection ps = templatePlotStyle != null? (G2DPlotStyleCollection)templatePlotStyle.Clone() : new G2DPlotStyleCollection();
bool foundMoreColumns = true;
for (idx = idx + 1; foundMoreColumns && idx < numColumns; idx++)
{
DataColumn col = table.DataColumns[idx];
switch (table.DataColumns.GetColumnKind(idx))
{
case ColumnKind.Label:
LabelPlotStyle labelStyle = new LabelPlotStyle(col);
ps.Insert(0, labelStyle);
break;
case ColumnKind.Err:
ErrorBarPlotStyle errStyle = new ErrorBarPlotStyle();
errStyle.PositiveErrorColumn = col as INumericColumn;
errStyle.NegativeErrorColumn = col as INumericColumn;
ps.Add(errStyle);
break;
case ColumnKind.pErr:
if (null != unpairedNegativeError)
{
unpairedNegativeError.PositiveErrorColumn = col as INumericColumn;;
unpairedNegativeError = null;
}
else
{
unpairedPositiveError = new ErrorBarPlotStyle();
unpairedPositiveError.PositiveErrorColumn = col as INumericColumn;
ps.Add(unpairedPositiveError);
}
break;
case ColumnKind.mErr:
if (null != unpairedPositiveError)
{
unpairedPositiveError.NegativeErrorColumn = col as INumericColumn;
unpairedPositiveError = null;
}
else
{
unpairedNegativeError = new ErrorBarPlotStyle();
unpairedNegativeError.NegativeErrorColumn = col as INumericColumn;
ps.Add(unpairedNegativeError);
}
break;
default:
foundMoreColumns = false;
break;
}
if (foundMoreColumns)
{
processedColumns.Add(idx);
}
}
result.Add(new XYColumnPlotItem(pa, ps));
}
return(result);
}
/// <summary>
/// Initializes a new instance of the <see cref="DataTableMultipleColumnProxy"/> class. The selected collections determine which columns and rows contribute to this instance.
/// The group number is determined by the first selected column (or, if no column is selected, by the first column of the data table).
/// </summary>
/// <param name="identifier">The identifier of the bundle of columns that are initially set with this constructor.</param>
/// <param name="table">The underlying table.</param>
/// <param name="selectedDataRows">The selected data rows.</param>
/// <param name="selectedDataColumns">The selected data columns.</param>
/// <exception cref="System.ArgumentNullException">table must not be null.</exception>
public DataTableMultipleColumnProxy(string identifier, DataTable table, IAscendingIntegerCollection selectedDataRows, IAscendingIntegerCollection selectedDataColumns)
{
if (null == identifier)
{
throw new ArgumentNullException("identifier");
}
if (null == table)
{
throw new ArgumentNullException("table");
}
_dataColumnBundles = new Dictionary <string, ColumnBundleInfo>();
_dataTable = new DataTableProxy(table)
{
ParentObject = this
};
_groupNumber = 0;
if (null != selectedDataColumns && selectedDataColumns.Count > 0)
{
_groupNumber = table.DataColumns.GetColumnGroup(table[selectedDataColumns[0]]);
}
var bundle = new ColumnBundleInfo();
_dataColumnBundles.Add(identifier, bundle);
int maxRowCount = 0;
if (selectedDataColumns != null && selectedDataColumns.Count > 0)
{
for (int i = 0; i < selectedDataColumns.Count; ++i)
{
var col = table[selectedDataColumns[i]];
if (table.DataColumns.GetColumnGroup(col) == _groupNumber)
{
InternalAddDataColumnNoClone(bundle, ReadableColumnProxyBase.FromColumn(col));
maxRowCount = Math.Max(maxRowCount, col.Count);
}
}
}
else // nothing selected - use all columns of group number 0
{
for (int i = 0; i < table.DataColumnCount; ++i)
{
var col = table[i];
if (table.DataColumns.GetColumnGroup(col) == _groupNumber)
{
InternalAddDataColumnNoClone(bundle, ReadableColumnProxyBase.FromColumn(col));
maxRowCount = Math.Max(maxRowCount, col.Count);
}
}
}
_useAllAvailableDataRows = null == selectedDataRows || selectedDataRows.Count == 0;
_participatingDataRows = new AscendingIntegerCollection(_useAllAvailableDataRows ? ContiguousIntegerRange.FromStartAndCount(0, maxRowCount) : selectedDataRows);
}
/// <summary>
/// Expand the source columns according to the provided options. The source table and the settings are provided in the <paramref name="options"/> variable.
/// The provided destination table is cleared from all data and property values before.
/// </summary>
/// <param name="inputData">The data containing the source table, the participating columns and the column with the cycling variable.</param>
/// <param name="options">The settings for expanding.</param>
/// <param name="destTable">The destination table. Any data will be removed before filling with the new data.</param>
/// <returns>Null if the method finishes successfully, or an error information.</returns>
public static string ExpandCyclingVariableColumn(DataTableMultipleColumnProxy inputData, ExpandCyclingVariableColumnOptions options, DataTable destTable)
{
var srcTable = inputData.DataTable;
try
{
ExpandCyclingVariableColumnDataAndOptions.EnsureCoherence(inputData, true);
}
catch (Exception ex)
{
return(ex.Message);
}
destTable.DataColumns.RemoveColumnsAll();
destTable.PropCols.RemoveColumnsAll();
DataColumn srcCycCol = inputData.GetDataColumnOrNull(ExpandCyclingVariableColumnDataAndOptions.ColumnWithCyclingVariableIdentifier);
var repeatRanges = DecomposeIntoRepeatUnits(srcCycCol);
// check if there is at least one averaged column
var columnsToAverageOverRepeatPeriod = inputData.GetDataColumns(ExpandCyclingVariableColumnDataAndOptions.ColumnsToAverageIdentifier);
if (options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.FirstAveragedColumn && columnsToAverageOverRepeatPeriod.Count == 0)
{
throw new ArgumentException("In order to let the first averaged column being the x-column, a column to average is needed, but the options didn't provide such column!");
}
// get the other columns to process
var srcColumnsToProcess = new List <DataColumn>(inputData.GetDataColumns(ExpandCyclingVariableColumnDataAndOptions.ColumnsParticipatingIdentifier));
// subtract cyclic variable column and average columns
srcColumnsToProcess.Remove(srcCycCol);
foreach (var col in columnsToAverageOverRepeatPeriod)
{
srcColumnsToProcess.Remove(col);
}
// --- Create and calculate the averaged columns, for now only temporarily ---
var propColsTemp = AverageColumns(srcTable, columnsToAverageOverRepeatPeriod, options, repeatRanges);
// --- avgValueOrder designates the ordering of the first averaged column and therefore of the sorting of the ranges and of the first averaged column
int[] avgValueOrder = Sorting.CreateIdentityIndices(repeatRanges.Count);
// --- prepare the sorting of columns by first averaged column ---
var rangeOrderSorting = options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.CyclingVariable ? options.DestinationColumnSorting : options.DestinationRowSorting;
if (propColsTemp.Length > 0 && rangeOrderSorting != ExpandCyclingVariableColumnOptions.OutputSorting.None)
{
avgValueOrder = Sorting.HeapSortVirtually(propColsTemp[0], avgValueOrder);
if (rangeOrderSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending)
{
Sorting.ReverseArray(avgValueOrder);
}
}
// prepare the sorting of the cycling values
var cycValueSorting = options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.CyclingVariable ? options.DestinationRowSorting : options.DestinationColumnSorting;
// create a dictionary with the cycling values (unique) and the corresponding ordering index
var cycValueOrder = GetUniqueValues(srcCycCol, cycValueSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending);
if (options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.CyclingVariable)
{
int[] propColsIdx = CreatePropColsForAveragedColumns(srcTable, columnsToAverageOverRepeatPeriod, options, destTable);
// --- Fill the x column, take the row sorting into account ---
var destXCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCycCol), srcCycCol.GetType(), ColumnKind.X, srcTable.DataColumns.GetColumnGroup(srcCycCol));
foreach (var entry in cycValueOrder)
{
destXCol[entry.Value] = entry.Key;
}
if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupOneColumn)
{
// foreach sourceColumnToProcess create as many destination columns as there are cycling ranges available
foreach (var srcCol in srcColumnsToProcess)
{
int nCreatedCol = -1;
var destColumnsToSort = new AscendingIntegerCollection();
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nCreatedCol;
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCreatedCol.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
destColumnsToSort.Add(nDestCol);
foreach (var nSrcRow in range)
{
int nDestRow = cycValueOrder[srcCycCol[nSrcRow]];
destCol[nDestRow] = srcCol[nSrcRow];
}
// fill also property columns
for (int nPropCol = 0; nPropCol < propColsTemp.Length; nPropCol++)
{
destTable.PropCols[propColsIdx[nPropCol]][nDestCol] = propColsTemp[nPropCol][rangeIndex];
}
}
} // repeat for each source colum to process
}
else if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupAllColumns)
{
int nCreatedCol = -1; // running number of processed range for column creation (Naming)
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nCreatedCol;
foreach (var srcCol in srcColumnsToProcess)
{
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCreatedCol.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
foreach (var nSrcRow in range)
{
int nDestRow = cycValueOrder[srcCycCol[nSrcRow]];
destCol[nDestRow] = srcCol[nSrcRow];
}
// fill also property columns
for (int nPropCol = 0; nPropCol < propColsTemp.Length; nPropCol++)
{
destTable.PropCols[propColsIdx[nPropCol]][nDestCol] = propColsTemp[nPropCol][rangeIndex];
}
}
}
}
else
{
throw new NotImplementedException("The option for destination output is unknown: " + options.DestinationOutput.ToString());
}
}
else if (options.DestinationX == ExpandCyclingVariableColumnOptions.DestinationXColumn.FirstAveragedColumn)
{
// now the first x column contains the values of the averaged column
// the rest of the data columns is repeated as many times as there are members in each repeat range
DataColumn srcXCol = columnsToAverageOverRepeatPeriod[0];
var destXCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcXCol), srcXCol.GetType(), ColumnKind.X, srcTable.DataColumns.GetColumnGroup(srcXCol));
// Fill with destination X
for (int nDestRow = 0; nDestRow < repeatRanges.Count; nDestRow++)
{
destXCol[nDestRow] = propColsTemp[0][avgValueOrder[nDestRow]];
}
// the only property column that is now usefull is that with the repeated values
var destPropCol = destTable.PropCols.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCycCol), srcCycCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCycCol), srcTable.DataColumns.GetColumnGroup(srcCycCol));
if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupOneColumn)
{
foreach (var srcCol in srcColumnsToProcess)
{
int nCurrNumber = -1;
IEnumerable <AltaxoVariant> cycValues = cycValueOrder.Keys;
if (options.DestinationColumnSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending)
{
cycValues = cycValueOrder.Keys.Reverse();
}
foreach (var cycValue in cycValues)
{
++nCurrNumber;
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCurrNumber.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
int nDestRow = -1;
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nDestRow;
int nSrcRow = FindSrcXRow(srcCycCol, cycValue, range);
if (nSrcRow >= 0)
{
destCol[nDestRow] = srcCol[nSrcRow];
}
}
// fill also property column
destPropCol[nDestCol] = cycValue;
}
}
}
else if (options.DestinationOutput == ExpandCyclingVariableColumnOptions.OutputFormat.GroupAllColumns)
{
IEnumerable <AltaxoVariant> positionsKeys = cycValueOrder.Keys;
if (options.DestinationColumnSorting == ExpandCyclingVariableColumnOptions.OutputSorting.Descending)
{
positionsKeys = cycValueOrder.Keys.Reverse();
}
int nCurrNumber = -1;
foreach (var xVal in positionsKeys)
{
++nCurrNumber;
foreach (var srcCol in srcColumnsToProcess)
{
var destCol = destTable.DataColumns.EnsureExistence(srcTable.DataColumns.GetColumnName(srcCol) + "." + nCurrNumber.ToString(), srcCol.GetType(), srcTable.DataColumns.GetColumnKind(srcCol), srcTable.DataColumns.GetColumnGroup(srcCol));
var nDestCol = destTable.DataColumns.GetColumnNumber(destCol);
int nDestRow = -1;
foreach (int rangeIndex in avgValueOrder)
{
var range = repeatRanges[rangeIndex];
++nDestRow;
int nSrcRow = FindSrcXRow(srcCycCol, xVal, range);
if (nSrcRow >= 0)
{
destCol[nDestRow] = srcCol[nSrcRow];
}
}
// fill also property column
destPropCol[nDestCol] = xVal;
}
}
}
else
{
throw new NotImplementedException("The option for destination output is unknown: " + options.DestinationOutput.ToString());
}
}
return(null);
}