public void Gradient_DifferentLayers_1Iter_CrossEntropy_Regularization()
{
// arrange
var activation = Activation.ReLU;
var net = new ConvNet(1, 5)
{
IsTraining = true
};
net.AddLayer(new ConvLayer(outputDepth: 2, windowSize: 3, padding: 1));
net.AddLayer(new MaxPoolingLayer(windowSize: 3, stride: 2, activation: Activation.Exp));
net.AddLayer(new ActivationLayer(activation: Activation.Tanh));
net.AddLayer(new FlattenLayer(outputDim: 10, activation: activation));
net.AddLayer(new DropoutLayer(rate: 0.5D));
net.AddLayer(new DenseLayer(outputDim: 3, activation: Activation.Exp));
net._Build();
net.RandomizeParameters(seed: 0);
var sample = new ClassifiedSample <double[][, ]>();
for (int i = 0; i < 3; i++)
{
var point = RandomPoint(1, 5, 5);
sample[point] = new Class(i.ToString(), i);
}
var regularizator = Regularizator.Composite(Regularizator.L1(0.1D), Regularizator.L2(0.3D));
var alg = new BackpropAlgorithm(net)
{
LearningRate = 0.1D,
LossFunction = Loss.CrossEntropySoftMax,
Regularizator = regularizator
};
alg.Build();
// act
var data = sample.First();
var expected = new double[3] {
1.0D, 0.0D, 0.0D
};
alg.RunIteration(data.Key, expected);
regularizator.Apply(alg.Gradient, alg.Net.Weights);
((DropoutLayer)alg.Net[4]).ApplyCustomMask = true;
// assert
AssertNetGradient(alg, data.Key, expected);
}
private DecisionNode <TObj> trainID3Core(IEnumerable <Predicate <TObj> > patterns, ClassifiedSample <TObj> sample, IInformativityIndex <TObj> informativity)
{
if (!sample.Any())
{
throw new MLException("Empty sample");
}
var cls = sample.First().Value;
if (sample.All(kvp => kvp.Value.Equals(cls)))
{
return(new LeafNode <TObj>(cls));
}
var pattern = informativity.Max(patterns, sample);
var negSample = new ClassifiedSample <TObj>();
var posSample = new ClassifiedSample <TObj>();
foreach (var pData in sample)
{
if (pattern(pData.Key))
{
posSample.Add(pData.Key, pData.Value);
}
else
{
negSample.Add(pData.Key, pData.Value);
}
}
if (!negSample.Any() || !posSample.Any())
{
var majorClass = sample.GroupBy(pd => pd.Value)
.Select(g => new KeyValuePair <Class, int>(g.Key, g.Count()))
.OrderByDescending(c => c.Value)
.First();
return(new LeafNode <TObj>(majorClass.Key));
}
var node = new InnerNode <TObj>(pattern);
var negNode = trainID3Core(patterns, negSample, informativity);
var posNode = trainID3Core(patterns, posSample, informativity);
node.SetNegativeNode(negNode);
node.SetPositiveNode(posNode);
return(node);
}
public void Gradient_MNISTSimple_1Iter()
{
// arrange
var activation = Activation.ReLU;
var net = new ConvNet(1, 14)
{
IsTraining = true
};
net.AddLayer(new ConvLayer(outputDepth: 4, windowSize: 5));
net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation));
net.AddLayer(new ConvLayer(outputDepth: 8, windowSize: 5));
net.AddLayer(new MaxPoolingLayer(windowSize: 2, stride: 2, activation: activation));
net.AddLayer(new FlattenLayer(outputDim: 10, activation: activation));
net._Build();
Randomize(net.Weights, -1.0D, 1.0D);
var sample = new ClassifiedSample <double[][, ]>();
for (int i = 0; i < 10; i++)
{
var point = RandomPoint(1, 14, 14);
sample[point] = new Class(i.ToString(), i);
}
var alg = new BackpropAlgorithm(net)
{
LearningRate = 0.005D,
LossFunction = Loss.Euclidean
};
alg.Build();
// act
var data = sample.First();
var expected = new double[10] {
1.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D
};
alg.RunIteration(data.Key, expected);
// assert
AssertNetGradient(alg, data.Key, expected);
}
public static int GetDimension(this ClassifiedSample <double[]> sample, bool cache = true)
{
return(sample.First().Key.Length);
}