protected AdaptiveDataTableAdaptorBase(ILinearAlgebraProvider lap,
ILearningContext learningContext, IDataTable dataTable) : base(lap, dataTable)
{
Debug.Assert(learningContext == null || learningContext.DeferUpdates);
_learningContext = learningContext;
_input = new InputFeeder(0);
}
/// <summary>
/// Creates a graph training engine
/// </summary>
/// <param name="dataSource">Data source with training data</param>
/// <param name="learningContext">Previously created training context</param>
/// <param name="graph">The serialised graph to execute (optional)</param>
/// <returns></returns>
public IGraphTrainingEngine CreateTrainingEngine(IDataSource dataSource,
ILearningContext learningContext, Models.ExecutionGraph graph = null)
{
var input = this.CreateFrom(graph);
return(new TrainingEngine(LinearAlgebraProvider, dataSource, learningContext, input));
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
using var previousVelocity = _cache.Clone();
_cache.AddInPlace(delta, _momentum);
previousVelocity.AddInPlace(_cache, -_momentum, 1 + _momentum);
_updater.Update(source, previousVelocity, context);
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
var l1 = context.BatchLearningRate * _lambda;
source.L1Regularisation(l1);
base.Update(source, delta, context);
}
public void UpdateBias(IMatrix delta, ILearningContext context)
{
using (var columnSums = delta.ColumnSums()) {
columnSums.Multiply(1f / delta.RowCount);
_bias.AddInPlace(columnSums, 1f, context.BatchLearningRate);
}
}
public SequenceToSequenceDataTableAdaptor(ILinearAlgebraProvider lap, ILearningContext learningContext, IDataTable dataTable, INode input, DataSourceModel dataSource)
: base(lap, learningContext, dataTable)
{
_Initialise(dataTable);
_input = input;
_inputSize = dataSource.InputSize;
_outputSize = dataSource.OutputSize;
}
private SequenceToSequenceDataTableAdaptor(ILinearAlgebraProvider lap, ILearningContext learningContext, IDataTable dataTable, INode input, int inputSize, int outputSize)
: base(lap, learningContext, dataTable)
{
_Initialise(dataTable);
_input = input;
_inputSize = inputSize;
_outputSize = outputSize;
}
public virtual void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
using var deltaSquared = delta.PointwiseMultiply(delta);
_cache.AddInPlace(deltaSquared);
using var cachedSqrt = _cache.Sqrt();
using var delta2 = delta.PointwiseDivide(cachedSqrt);
_updater.Update(source, delta2, context);
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
using (var deltaSquared = delta.PointwiseMultiply(delta)) {
_cache.AddInPlace(deltaSquared, _decayRate, 1 - _decayRate);
using (var cachedSqrt = _cache.Sqrt(1e-8f))
using (var delta2 = delta.PointwiseDivide(cachedSqrt)) {
_updater.Update(source, delta2, context);
}
}
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
var t = context.CurrentEpoch;
using var deltaSquared = delta.PointwiseMultiply(delta);
_cache.AddInPlace(delta, _decayRate, 1 - _decayRate);
_cache2.AddInPlace(deltaSquared, _decayRate2, 1 - _decayRate2);
using var mb = _cache.Clone();
using var vb = _cache2.Clone();
mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decayRate, t))));
vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decayRate2, t))));
using var vbSqrt = vb.Sqrt();
using var delta2 = mb.PointwiseDivide(vbSqrt);
_updater.Update(source, delta2, context);
}
public SequenceToSequenceDataTableAdaptor(ILinearAlgebraProvider lap, ILearningContext learningContext, GraphFactory factory, IDataTable dataTable, Action <WireBuilder> dataConversionBuilder)
: base(lap, learningContext, dataTable)
{
_Initialise(dataTable);
var wireBuilder = factory.Connect(_inputSize, _input);
dataConversionBuilder(wireBuilder);
// execute the graph to find the input size (which is the size of the adaptive graph's output)
using (var executionContext = new ExecutionContext(lap)) {
var output = _Encode(executionContext, new[] { 0 });
_inputSize = output.Item1.ColumnCount;
_learningContext.Clear();
}
}
/// <summary>
/// Creates an adaptive data source (that uses the output from a preliminary graph)
/// </summary>
/// <param name="dataTable">Data that will be sent to the preliminary graph</param>
/// <param name="learningContext">Learning context to use while training the preliminary graph</param>
/// <param name="dataConversionBuilder">Callback to build the preliminary graph</param>
/// <returns></returns>
public IDataSource CreateDataSource(IDataTable dataTable, ILearningContext learningContext, Action <WireBuilder> dataConversionBuilder)
{
var columns = dataTable.Columns;
if (columns.Count == 2)
{
var column1 = columns[0].Type;
var column2 = columns[1].Type;
// sequence to sequence
if (column1 == ColumnType.Matrix && column2 == ColumnType.Matrix)
{
return(new SequenceToSequenceDataTableAdaptor(_lap, learningContext, this, dataTable, dataConversionBuilder));
}
}
throw new ArgumentException($"{nameof(dataTable)} does not contain a recognised data format");
}
/// <summary>
/// Creates an adaptive data source from a serialised model
/// </summary>
/// <param name="dataTable">Data that will be sent to the preliminary graph</param>
/// <param name="dataSource">The serialised preliminary graph</param>
/// <param name="learningContext">Learning context to use while training the preliminary graph</param>
/// <returns></returns>
public IDataSource CreateDataSource(IDataTable dataTable, DataSourceModel dataSource,
ILearningContext learningContext = null)
{
var input = this.CreateFrom(dataSource.Graph);
var columns = dataTable.Columns;
if (columns.Count == 2)
{
var column1 = columns[0].Type;
var column2 = columns[1].Type;
if (column1 == ColumnType.Matrix && column2 == ColumnType.Matrix)
{
return(new SequenceToSequenceDataTableAdaptor(LinearAlgebraProvider, learningContext,
dataTable, input, dataSource));
}
}
throw new ArgumentException($"{nameof(dataTable)} does not contain a recognised data format");
}
public void UpdateBias(IMatrix delta, ILearningContext context)
{
using var columnSums = delta.ColumnSums();
_bias.AddInPlace(columnSums, 1f / columnSums.Count, context.BatchLearningRate);
}
protected void _Update(IMatrix source, IMatrix delta, ILearningContext context, float coefficient1, float coefficient2)
{
source.AddInPlace(delta, coefficient1, coefficient2);
}
public void UpdateWeights(IMatrix delta, ILearningContext context)
{
_updater.Update(_weight, delta, context);
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
var l2 = 1.0f - (context.BatchLearningRate * _lambda);
_Update(source, delta, context, l2, context.BatchLearningRate);
}
public override void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
_cache.AddInPlace(delta, 1f, _momentum);
_updater.Update(source, _cache, context);
}
public void Update(IMatrix delta, ILearningContext context)
{
_updater.Update(_filter, delta, context);
}
void _UpdateBias(IVector delta, ILearningContext context)
{
_source._bias.AddInPlace(delta, 1f, context.BatchLearningRate);
}
public virtual void Update(IMatrix source, IMatrix delta, ILearningContext context)
{
_Update(source, delta, context, 1f, context.BatchLearningRate);
}
