private torch.Tensor ForwardEmbedding(torch.Tensor tokens, torch.Tensor segmentLabels, torch.Tensor positions) { using var disposeScope = torch.NewDisposeScope(); var x = TokenEmbedding.forward(tokens); if (EmbedScale != null) { x.mul_(EmbedScale); } if (PositionalEmbedding != null) { var positionalEmbedding = PositionalEmbedding.forward(tokens, new Dictionary <string, object> { { PositionalEmbedding.PositionKey, positions } }); x.add_(positionalEmbedding); } if (SegmentEmbedding != null && segmentLabels.IsNotNull()) { var segmentEmbedding = SegmentEmbedding.forward(segmentLabels); x.add_(segmentEmbedding); } if (EmbeddingLayerNorm != null) { x = EmbeddingLayerNorm.forward(x); } x = EmbedTransfer.forward(x, (int)x.size()[x.size().Length - 1]); x = DropoutLayer.forward(x); return(x.MoveToOuterDisposeScope()); }
public override torch.Tensor forward(torch.Tensor x, Dictionary <string, object> param) { using var disposeScope = torch.NewDisposeScope(); if (!ParseArguments(param, out var selfAttentionMask, out var selfAttentionPaddingMask)) { throw new ArgumentException("Invalid arguments."); } var attention = SelfAttention.forward(query: x, key: x, value: x, out _, keyPaddingMask: selfAttentionPaddingMask, needWeights: false, attentionMask: selfAttentionMask); var dropout = DropoutLayer.forward(attention); dropout.add_(x); var norm = LayerNorm.forward(dropout); return(norm.MoveToOuterDisposeScope()); }
public override torch.Tensor forward(torch.Tensor x, Dictionary <string, object> param) { using var layerOutput = FullConnects.forward(x); using var layerOuptutIntermediate = layerOutput.add_(x); return(FinalLayerNorm.forward(layerOutput)); }