public IDataTable Normalise(NormalisationType normalisationType, Stream output = null)
{
var normaliser = new DataTableNormaliser(this, normalisationType, output);
Process(normaliser);
return(normaliser.GetDataTable());
}
DataTableNormalisation.Column _GetColumn(NormalisationType type, INumericColumnInfo numericInfo, int columnIndex, ColumnType columnType)
{
if (type == NormalisationType.Standard && !numericInfo.StdDev.HasValue)
{
return(null);
}
if (type == NormalisationType.Standard)
{
return(new DataTableNormalisation.Column(columnIndex, columnType, numericInfo.StdDev ?? 1, numericInfo.Mean));
}
else if (type == NormalisationType.Euclidean)
{
return(new DataTableNormalisation.Column(columnIndex, columnType, numericInfo.L2Norm));
}
else if (type == NormalisationType.Manhattan)
{
return(new DataTableNormalisation.Column(columnIndex, columnType, numericInfo.L1Norm));
}
else if (type == NormalisationType.FeatureScale)
{
return(new DataTableNormalisation.Column(columnIndex, columnType, numericInfo.Max - numericInfo.Min, numericInfo.Min));
}
else
{
throw new NotImplementedException();
}
}
/// <summary>
/// Sets the axes to the specified values
/// </summary>
void SetupAxes(axisParameters horizontalAxis, axisParameters verticalAxis, NormalisationType normalisation)
{
this.horizontalAxis = horizontalAxis;
this.verticalAxis = verticalAxis;
graphNormalisationType = normalisation;
Invalidate();
}
public void Normalise(NormalisationType type)
{
if (type == NormalisationType.FeatureScale)
{
var minMax = GetMinMax();
float range = minMax.Max - minMax.Min;
if (range > 0)
{
_vector.MapInplace(v => (v - minMax.Min) / range);
}
}
else if (type == NormalisationType.Standard)
{
var mean = Average();
var stdDev = StdDev(mean);
if (stdDev != 0)
{
_vector.MapInplace(v => (v - mean) / stdDev);
}
}
else if (type == NormalisationType.Euclidean || type == NormalisationType.Manhattan)
{
var p = (type == NormalisationType.Manhattan) ? 1.0 : 2.0;
var norm = _vector.Normalize(p);
norm.CopyTo(_vector);
}
}
public IDataTable Normalise(NormalisationType normalisationType, Stream output = null, IEnumerable <int> columnIndices = null)
{
var normaliser = new DataTableNormaliser(this, normalisationType, output, columnIndices);
Process(normaliser);
return(normaliser.GetDataTable());
}
public void Normalise(NormalisationType type)
{
Debug.Assert(IsValid);
if (type == NormalisationType.FeatureScale)
{
var minMax = GetMinMax();
float range = minMax.Max - minMax.Min;
if (range > 0)
{
_cuda.Normalise(Memory, Count, minMax.Min, range);
}
}
else if (type == NormalisationType.Standard)
{
var mean = Average();
var stdDev = StdDev(mean);
if (BoundMath.IsNotZero(stdDev))
{
_cuda.Normalise(Memory, Count, mean, stdDev);
}
}
else if (type == NormalisationType.Euclidean || type == NormalisationType.Manhattan)
{
var p = type == NormalisationType.Manhattan ? L1Norm() : L2Norm();
if (BoundMath.IsNotZero(p))
{
Multiply(1f / p);
}
}
}
public DataTableNormaliser(IDataTable dataTable, NormalisationType type, Stream output, IEnumerable <int> columnIndices)
{
_writer = new DataTableWriter(dataTable.Columns, output);
var analysis = dataTable.GetAnalysis();
var columnNormList = new List <DataTableNormalisation.Column>();
var columns = analysis.ColumnInfo.AsQueryable();
if (columnIndices != null)
{
var columnSet = new HashSet <int>(columnIndices);
columns = columns.Where(ci => columnSet.Contains(ci.ColumnIndex));
}
var vectorColumns = new List <(int ColumnIndex, int Size)>();
foreach (var columnInfo in columns)
{
var column = dataTable.Columns[columnInfo.ColumnIndex];
if (column.IsContinuous && columnInfo is INumericColumnInfo numericInfo)
{
var columnNorm = _GetColumn(type, numericInfo, columnInfo.ColumnIndex, column.Type);
if (columnNorm != null)
{
columnNormList.Add(columnNorm);
}
}
else if (column.Type == ColumnType.Vector && columnInfo is IDimensionsColumnInfo vector && vector.XDimension.HasValue && vector.XDimension.Value > 0)
{
vectorColumns.Add((column.Index, vector.XDimension.Value));
}
}
DataTableNormalisation.VectorColumn[] vectorColumnNormList = null;
if (vectorColumns.Any())
{
var collectors = vectorColumns.Select(vc => Enumerable.Range(0, vc.Size).Select(i => new NumberCollector(i)).ToList()).ToList();
dataTable.ForEach(row => {
foreach (var column in vectorColumns.Zip(collectors, (vc, c) => (vc, c)))
{
var vectorAsRow = row.GetField <FloatVector>(column.Item1.ColumnIndex).AsRow();
foreach (var collector in column.Item2)
{
collector.Process(vectorAsRow);
}
}
});
vectorColumnNormList = collectors.Select((c, i) => new DataTableNormalisation.VectorColumn {
ColumnIndex = vectorColumns[i].ColumnIndex,
VectorColumns = c.Select((nc, j) => _GetColumn(type, nc, j, ColumnType.Float)).ToArray()
}).ToArray();
}
_normalisationModel = new DataTableNormalisation {
Type = type,
ColumnNormalisation = columnNormList.ToArray(),
VectorColumnNormalisation = vectorColumnNormList
};
}
public DataTableNormaliser(IDataTable dataTable, NormalisationType type, Stream output = null, DataTableNormalisation model = null)
{
_table = dataTable;
_writer = new DataTableWriter(dataTable.Columns, output);
if (model != null)
{
_normalisationModel = model;
}
else
{
var analysis = dataTable.GetAnalysis();
var columnNormList = new List <DataTableNormalisation.Column>();
foreach (var columnInfo in analysis.ColumnInfo)
{
var column = dataTable.Columns[columnInfo.ColumnIndex];
if (column.IsContinuous)
{
var numericInfo = columnInfo as INumericColumnInfo;
if (numericInfo != null)
{
if (type == NormalisationType.Standard && !numericInfo.StdDev.HasValue)
{
continue;
}
DataTableNormalisation.Column columnNorm;
if (type == NormalisationType.Standard)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.StdDev.Value, numericInfo.Mean);
}
else if (type == NormalisationType.Euclidean)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.L2Norm);
}
else if (type == NormalisationType.Manhattan)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.L1Norm);
}
else if (type == NormalisationType.FeatureScale)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.Max - numericInfo.Min, numericInfo.Min);
}
else
{
throw new NotImplementedException();
}
columnNormList.Add(columnNorm);
}
}
}
_normalisationModel = new DataTableNormalisation {
Type = type,
ColumnNormalisation = columnNormList.ToArray()
};
}
}
public override void SetupAxes()
{
base.SetupAxes();
if (!AxesUserModified)
{
normalisationType = NormalisationType.None;
}
}
public DataTableNormaliser(IDataTable dataTable, NormalisationType type, Stream output, IEnumerable <int> columnIndices)
{
_writer = new DataTableWriter(dataTable.Columns, output);
var analysis = dataTable.GetAnalysis();
var columnNormList = new List <DataTableNormalisation.Column>();
var columns = analysis.ColumnInfo.AsQueryable();
if (columnIndices != null)
{
var columnSet = new HashSet <int>(columnIndices);
columns = columns.Where(ci => columnSet.Contains(ci.ColumnIndex));
}
foreach (var columnInfo in columns)
{
var column = dataTable.Columns[columnInfo.ColumnIndex];
if (column.IsContinuous)
{
if (columnInfo is INumericColumnInfo numericInfo)
{
if (type == NormalisationType.Standard && !numericInfo.StdDev.HasValue)
{
continue;
}
DataTableNormalisation.Column columnNorm;
if (type == NormalisationType.Standard)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.StdDev ?? 1, numericInfo.Mean);
}
else if (type == NormalisationType.Euclidean)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.L2Norm);
}
else if (type == NormalisationType.Manhattan)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.L1Norm);
}
else if (type == NormalisationType.FeatureScale)
{
columnNorm = new DataTableNormalisation.Column(columnInfo.ColumnIndex, column.Type, numericInfo.Max - numericInfo.Min, numericInfo.Min);
}
else
{
throw new NotImplementedException();
}
columnNormList.Add(columnNorm);
}
}
}
_normalisationModel = new DataTableNormalisation {
Type = type,
ColumnNormalisation = columnNormList.ToArray()
};
}
/// <summary>
/// Validates the data in the text boxes,
/// and populates the local axis data with the data from the text boxes
/// </summary>
/// <param name="incorrectValue">Returns true if any one of the data items is invalid</param>
void PopulateAxisValues(ref bool incorrectValue)
{
localHorizontalAxis.baseOffset = (int)Functions.ValidateBetweenInclusive(Convert.ToInt32(axisParameterDataBoxes[0, 0].Text), 0, Data.GetRaceLaps(), "Horizontal Axis Base Offset", ref incorrectValue, true);
localHorizontalAxis.scaleFactor = Functions.ValidateGreaterThanEqualTo(Convert.ToSingle(axisParameterDataBoxes[0, 1].Text), 0, "Horizontal Axis Scale", ref incorrectValue, true);
localHorizontalAxis.startLocation = (int)Functions.ValidateGreaterThan(Convert.ToInt32(axisParameterDataBoxes[0, 2].Text), 0, "Horizontal Axis Start", ref incorrectValue, true);
localHorizontalAxis.axisLabelSpacing = (int)Functions.ValidateGreaterThan(Convert.ToInt32(axisParameterDataBoxes[0, 3].Text), 1, "Horizontal Axis Label Spacing", ref incorrectValue, true);
localVerticalAxis.baseOffset = (int)Functions.ValidateBetweenInclusive(Convert.ToInt32(axisParameterDataBoxes[1, 0].Text), -75, 75, "Vertical Axis Base Offset", ref incorrectValue, true);
localVerticalAxis.scaleFactor = Functions.ValidateGreaterThanEqualTo(Convert.ToSingle(axisParameterDataBoxes[1, 1].Text), 0, "Vertical Axis Scale", ref incorrectValue, true);
localVerticalAxis.startLocation = (int)Functions.ValidateGreaterThan(Convert.ToInt32(axisParameterDataBoxes[1, 2].Text), 0, "Vertical Axis Start", ref incorrectValue, true);
localVerticalAxis.axisLabelSpacing = (int)Functions.ValidateGreaterThan(Convert.ToInt32(axisParameterDataBoxes[1, 3].Text), 1, "Vertical Axis Label Spacing", ref incorrectValue, true);
localNormalisationType = (NormalisationType)NormalisationTypeSelected.SelectedIndex;
}
//TODO: refactor to move parts from here into Graph, then this will inherit from graph.
public StrategyGraph(NormalisationType normalisationType, MainForm formToLink)
: base("Graph", formToLink, Properties.Resources.Graph)
{
SetRaceLaps(Data.GetRaceLaps());
lapNumber = raceLaps;
graphPanel.SetNormalisationType(normalisationType);
PanelClosed += StrategyGraph_PanelClosed;
PanelOpened += StrategyGraph_PanelOpened;
SetupAxes(raceLaps);
SetPanelProperties(DockTypes.Top2, AutosizeTypes.Free, FillStyles.None, this.Size);
MyPadding = new Padding(5, 25, 5, 5);
}
void _TestNormalise(NormalisationType type)
{
var distribution = new Normal(0, 5);
IIndexableVector v2;
var a = _cpu.CreateVector(5000, i => Convert.ToSingle(distribution.Sample()));
using (var gpuA = _cuda.CreateVector(a.AsIndexable())) {
gpuA.Normalise(type);
v2 = gpuA.AsIndexable();
}
a.Normalise(type);
var v1 = a.AsIndexable();
FloatingPointHelper.AssertEqual(v1, v2, 12);
}
public void Normalise(NormalisationType type)
{
Debug.Assert(IsValid);
if (type == NormalisationType.FeatureScale)
{
var minMax = GetMinMax();
float range = minMax.Max - minMax.Min;
if (range > 0)
{
_cuda.Normalise(_data, _size, minMax.Min, range);
}
}
else if (type == NormalisationType.Standard)
{
var mean = Average();
var stdDev = StdDev(mean);
if (stdDev != 0)
{
_cuda.Normalise(_data, _size, mean, stdDev);
}
}
else if (type == NormalisationType.Euclidean || type == NormalisationType.Manhattan)
{
float p = 0f;
if (type == NormalisationType.Manhattan)
{
p = L1Norm();
}
else
{
p = L2Norm();
}
if (p != 0f)
{
Multiply(1f / p);
}
}
}
void NormalisedGraph_NormalisationTypeChanged(object sender, NormalisationType e)
{
Invalidate();
}
void MyEvents_AxesModified(axisParameters XAxis, axisParameters YAxis, NormalisationType normalisation)
{
localHorizontalAxis = XAxis;
localVerticalAxis = YAxis;
PopulateAxesBoxes(XAxis, YAxis, normalisation);
}
protected void MyEvents_AxesModified(axisParameters XAxis, axisParameters YAxis, NormalisationType normalisation, bool UserModified)
{
if (UserModified)
{
SetupAxes(XAxis, YAxis, normalisation);
}
}
public Graph(string graphTitle, NormalisationType normalisationType, MainForm formToLink, Image icon)
: base(500, 400, graphTitle, formToLink, icon)
{
normalisationType = graphNormalisationType;
}
/// <summary>
/// Fires the AxesChangedByComputer event. This represents any scaling or setting which does not involve the axes panel.
/// </summary>
/// <param name="horizontalAxis">The new horizontal axis</param>
/// <param name="verticalAxis">The new vertical axis</param>
/// <param name="normalisation">The new graph normalisation type</param>
public static void OnAxesComputerGenerated(axisParameters horizontalAxis, axisParameters verticalAxis, NormalisationType normalisation)
{
if (AxesChangedByComputer != null)
{
AxesChangedByComputer(horizontalAxis, verticalAxis, normalisation);
}
}
public void SetNormalisationType(NormalisationType normalisationType)
{
this.normalisationType = normalisationType;
AxesUserModified = true;
}
public DataTableNormalisation GetNormalisationModel(NormalisationType normalisationType)
{
var normaliser = new DataTableNormaliser(this, normalisationType);
return(normaliser.GetNormalisationModel());
}
public DataTableNormalisation GetNormalisationModel(NormalisationType normalisationType, IEnumerable <int> columnIndices = null)
{
var normaliser = new DataTableNormaliser(this, normalisationType, null, columnIndices);
return(normaliser.GetNormalisationModel());
}
private void MyEvents_AxesModifiedByUser(axisParameters horizontalAxis, axisParameters verticalAxis, NormalisationType normalisation)
{
GraphPanel.SetNormalisationType(normalisation);
GraphPanel.SetHorizontalAxis(horizontalAxis);
GraphPanel.SetVerticalAxis(verticalAxis);
Invalidate();
}
/// <summary>
/// Populates the text boxes on the panel with data from the given axes
/// </summary>
void PopulateAxesBoxes(axisParameters horizontalAxis, axisParameters verticalAxis, NormalisationType normalisationType)
{
axisParameterDataBoxes[0, 0].Text = Convert.ToString(horizontalAxis.baseOffset);
axisParameterDataBoxes[0, 1].Text = Convert.ToString(horizontalAxis.scaleFactor);
axisParameterDataBoxes[0, 2].Text = Convert.ToString(horizontalAxis.startLocation);
axisParameterDataBoxes[0, 3].Text = Convert.ToString(horizontalAxis.axisLabelSpacing);
axisParameterDataBoxes[1, 0].Text = Convert.ToString(verticalAxis.baseOffset);
axisParameterDataBoxes[1, 1].Text = Convert.ToString(verticalAxis.scaleFactor);
axisParameterDataBoxes[1, 2].Text = Convert.ToString(verticalAxis.startLocation);
axisParameterDataBoxes[1, 3].Text = Convert.ToString(verticalAxis.axisLabelSpacing);
NormalisationTypeSelected.SelectedIndex = (int)normalisationType;
}