public MLPVar(Config conf, int nInput, int nOuput, OptBase opt) : base(conf, nInput, nOuput, opt)
{
Layers = new DenseUnit[Conf.Layers];
for (var i = 0; i < Layers.Length; i++)
{
if (i == 0 && i == Conf.Layers - 1)
{
Layers[i] = new DenseUnit(this, InDim, OutDim);
}
else if (i == 0)
{
Layers[i] = new DenseMaskedUnit(this, InDim, HidDim);
}
else if (i == Conf.Layers - 1)
{
Layers[i] = new DenseUnit(this, HidDim, OutDim);
}
else
{
Layers[i] = new DenseMaskedUnit(this, HidDim, HidDim);
}
}
_top = conf.TrainTop;
}
public MLP(Config conf, int nInput, OptBase opt) : base(conf, nInput, opt)
{
var r = new RNG(conf.RandomSeed);
var layer = conf.Layer;
var nOutput = conf.LabelCount;
var nHidden = conf.HiddenSize;
_h = new DenseUnit[conf.Layer];
if (conf.Layer == 1)
{
_h[0] = new DenseUnit(this, nInput, nOutput, r);
}
else
{
for (var i = 0; i < conf.Layer; i++)
{
if (i == 0)
{
_h[i] = new DenseUnit(this, nInput, nHidden, r);
}
else if (i == layer - 1)
{
_h[i] = new DenseUnit(this, nHidden, nOutput, r);
}
else
{
_h[i] = new DenseUnit(this, nHidden, nHidden, r);
}
}
}
keep = 1.0 - conf.DropProb;
}
public void Recover(Config conf, DataSet train)
{
_train = train;
_conf = conf;
FixedParams.AddRange(_surface.SubmitParameters());
FixedParams.AddRange(_act.SubmitParameters());
FixedParams.AddRange(_label.SubmitParameters());
FixedParams.AddRange(_actOutput.SubmitParameters());
FixedParams.AddRange(_labelOutput.SubmitParameters());
FixedParams.Add(_non);
AllParams.AddRange(FixedParams);
AllParams.AddRange(_formemb);
AllParams.AddRange(_posemb);
switch (_conf.OptType)
{
case OptType.sgd:
_opt = new Sgd(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, _conf.DecayRate,
train.Count);
break;
case OptType.adagrad:
_opt = new AdaGrad(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, _conf.Eps);
break;
case OptType.adam:
_opt = new Adam(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, 0.999f, 0.9f, 1e-8f);
break;
default:
throw new ArgumentOutOfRangeException(nameof(_conf.OptType), "unknown opt type");
}
}
protected NetBase(Config conf, int idim, OptBase opt)
{
_opt = opt;
InDim = idim;
OutDim = conf.LabelCount;
HidDim = conf.HiddenSize;
FixedParam = new List <Tensor>();
_seed = conf.RandomSeed;
}
public MLP(Config conf, DataSet train)
{
_train = train;
_conf = conf;
_rng = new RandomNumberGenerator(conf.Seed);
_rngChoice = new RandomNumberGenerator(conf.Seed);
_formemb = File.Exists(conf.EmbedFile)
? InitEmbed(conf.EmbedFile, train.Form, conf.InitRange)
: InitEmbed(train.Form, conf.InitRange);
_posemb = InitEmbed(train.PosTag, conf.InitRange);
_non = new Tensor(1, conf.HiddenSize * 2, true);
_surface = new BiLstmUnit(conf.EmbeddingSize * 2, conf.HiddenSize);
_act = new DenseUnit(conf.HiddenSize * 6, conf.HiddenSize);
_actOutput = new DenseUnit(conf.HiddenSize, 3);
_label = new DenseUnit(conf.HiddenSize * 6, conf.HiddenSize);
_labelOutput = new DenseUnit(conf.HiddenSize * 3, train.DepLabel.Count);
FixedParams.AddRange(_surface.SubmitParameters());
FixedParams.AddRange(_act.SubmitParameters());
FixedParams.AddRange(_label.SubmitParameters());
FixedParams.AddRange(_actOutput.SubmitParameters());
FixedParams.AddRange(_labelOutput.SubmitParameters());
FixedParams.Add(_non);
AllParams.AddRange(FixedParams);
AllParams.AddRange(_formemb);
AllParams.AddRange(_posemb);
_surface.Init((fin, fout) => new GlorotNormalInit(fin, fout, _rng).Next);
_act.Init((fin, fout) => new GlorotNormalInit(fin, fout, _rng).Next);
_label.Init((fin, fout) => new GlorotNormalInit(fin, fout, _rng).Next);
_actOutput.Init((fin, fout) => new GlorotNormalInit(fin, fout, _rng).Next);
_labelOutput.Init((fin, fout) => new GlorotNormalInit(fin, fout, _rng).Next);
switch (conf.OptType)
{
case OptType.sgd:
_opt = new Sgd(conf.LearningRate, conf.L2RegFactor, conf.ClipBound, conf.DecayRate,
train.Count);
break;
case OptType.adagrad:
_opt = new AdaGrad(conf.LearningRate, conf.L2RegFactor, conf.ClipBound, conf.Eps);
break;
case OptType.adam:
_opt = new Adam(conf.LearningRate, conf.L2RegFactor, conf.ClipBound, 0.999f, 0.9f, 1e-8f);
break;
default:
throw new ArgumentOutOfRangeException(nameof(conf.OptType), "unknown opt type");
}
_opt.Prepare(AllParams);
G.SetRng(conf.Seed);
}
private bool _isTraining; // the state of the current model; if false, no gradient operation will be recorded
protected NetBase(Config conf, int idim, int odim, OptBase opt)
{
_opt = opt;
Conf = conf;
InDim = idim;
OutDim = odim;
HidDim = Conf.HiddenSize;
FixedParam = new List <Tensor>();
Rand = new RandomNumberGenerator(conf.RandomSeed);
_isTraining = false;
}
private void InitNet(NetType netType, OptType optType)
{
switch (optType)
{
case OptType.sgd:
Opt = new Sgd(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, _conf.DecayRate,
_dataset.Examples.Count);
break;
case OptType.adagrad:
Opt = new AdaGrad(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, _conf.Eps);
break;
case OptType.adam:
Opt = new Adam(l2RegFactor: _conf.L2RegFactor, clipRange: _conf.ClipBound);
break;
default:
throw new ArgumentOutOfRangeException(nameof(optType), optType, null);
}
var eDim = _dataset.Examples[0].Feature.Length;
var hDim = _conf.HiddenSize;
var oDim = _conf.LabelCount;
switch (netType)
{
case NetType.mlp:
Net = new MLP(_conf, eDim, Opt);
break;
case NetType.mlptop:
Net = new MLPTop(_conf, eDim, Opt);
break;
case NetType.mlprand:
Net = new MLPRand(_conf, eDim, Opt);
break;
default:
throw new ArgumentOutOfRangeException(nameof(netType), netType, null);
}
}
private void InitNet(NetType netType, OptType optType)
{
switch (optType)
{
case OptType.sgd:
Opt = new Sgd(Global.Config.LearningRate, Global.Config.L2RegFactor, Global.Config.ClipBound, Global.Config.DecayRate,
_dataset.Examples.Count);
break;
case OptType.adagrad:
Opt = new AdaGrad(Global.Config.LearningRate, Global.Config.L2RegFactor, Global.Config.ClipBound, Global.Config.Eps);
break;
case OptType.adam:
Opt = new Adam(Global.Config.LearningRate, Global.Config.L2RegFactor, Global.Config.ClipBound, 0.999f, 0.9f, 1e-8f);
break;;
default:
throw new ArgumentOutOfRangeException(nameof(optType), optType, null);
}
var eDim = Global.Config.TokenCount * Global.Config.EmbeddingSize;
var hDim = Global.Config.HiddenSize;
var oDim = _labelCount;
switch (netType)
{
case NetType.ffnn:
Net = new FFNN(eDim, hDim, oDim, Opt);
break;
case NetType.blstm:
Net = new Blstm(eDim, hDim, oDim, Opt);
break;
default:
throw new ArgumentOutOfRangeException(nameof(netType), netType, null);
}
Net.InitEmbed(_embedd);
}
// build a optimizer
private void InitOpt(OptType optType)
{
switch (optType)
{
case OptType.sgd:
Opt = new Sgd(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, _conf.DecayRate,
_dataset.Examples.Count);
break;
case OptType.adagrad:
Opt = new AdaGrad(_conf.LearningRate, _conf.L2RegFactor, _conf.ClipBound, _conf.Eps);
break;
case OptType.adam:
Opt = new Adam(l2RegFactor: _conf.L2RegFactor, clipRange: _conf.ClipBound);
break;
default:
// should never reach this
throw new ArgumentOutOfRangeException(nameof(optType), optType, null);
}
}
public MLPTop(Config conf, int nInput, OptBase opt) : base(conf, nInput, opt)
{
_k = conf.TrainTop;
}
public MLPRand(Config conf, int nInput, int nOuput, OptBase opt) : base(conf, nInput, nOuput, opt)
{
_k = conf.TrainTop;
}
