public TransformerCellDiscrete(
int arch,
double dropout = 0.1,
double attentionDropout = 0.1,
double activationDropout = 0.1,
string activationFn = "relu",
bool addBiasKv = false,
bool addZeroAttention = false,
bool dynamicDropout = false)
: base(nameof(TransformerCellDiscrete))
{
_activationFn = new ActivationFunction(activationFn);
Operation = SearchSpace.GetLayer(arch, dropout, attentionDropout, activationDropout, activationFn, addBiasKv,
addZeroAttention, dynamicDropout);
RegisterComponents();
}
public TransformerCellNonDiscrete(
float dropout = 0.1f,
float attentionDropout = 0.1f,
float activationDropout = 0.1f,
string activationFn = "relu",
bool addBiasKv = false,
bool addZeroAttention = false,
bool dynamicDropout = false)
: base(nameof(TransformerCellNonDiscrete))
{
_activationFn = new ActivationFunction(activationFn);
var operations = Enumerable.Range(0, SearchSpace.NumLayerChoices)
.Select(i => SearchSpace.GetLayer(i, dropout, attentionDropout, activationDropout, activationFn,
addBiasKv, addZeroAttention, dynamicDropout) as torch.nn.Module)
.ToArray();
Operations = new ModuleList(operations);
RegisterComponents();
}
