/// <summary>
/// Writes this layer as XML
/// </summary>
/// <param name="writer">The XML writer</param>
public void WriteTo(XmlWriter writer)
{
writer.WriteStartElement("layer");
writer.WriteAttributeString("input-size", InputSize.ToString());
writer.WriteAttributeString("output-size", OutputSize.ToString());
writer.WriteAttributeString("activation", Activation.ToString());
writer.WriteAttributeString("weight-init", WeightInitialisation.ToString());
writer.WriteAttributeString("regularisation", Regularisation.ToString());
writer.WriteAttributeString("weight-update", WeightUpdate.ToString());
writer.WriteAttributeString("trainer", LayerTrainer.ToString());
writer.WriteAttributeString("lambda", Lambda.ToString());
writer.WriteAttributeString("momentum", Momentum.ToString());
writer.WriteAttributeString("decay-rate", DecayRate.ToString());
writer.WriteAttributeString("decay-rate2", DecayRate2.ToString());
writer.WriteAttributeString("dropout", Dropout.ToString());
if (Bias != null)
{
Bias.WriteTo("bias", writer);
}
if (Weight != null)
{
writer.WriteStartElement("weight");
foreach (var item in Weight)
{
item.WriteTo("row", writer);
}
writer.WriteEndElement();
}
writer.WriteEndElement();
}
/// <summary>
///
/// </summary>
/// <param name="inputSize">输入数据单个的size</param>
/// <param name="hiddenSize">隐藏层size</param>
/// <param name="outputSize">输出层size</param>
/// <param name="weightInit"></param>
public TwoLayerNet(int inputSize, int hiddenSize, int outputSize, double weightInit = 0.01)
{
Params = new Matrix[4];
grads = new Matrix[4];
layers = new Dictionary <string, ILayer>();
//初始化 权重 和 偏置
Params[0] = (weightInit * CommonFunctions.StandardNormalDistributionRondam(inputSize, hiddenSize));
Params[1] = (new Matrix(1, hiddenSize));
Params[2] = (weightInit * CommonFunctions.StandardNormalDistributionRondam(hiddenSize, outputSize));
Params[3] = (new Matrix(1, outputSize));
///第一层线性组合
affineLayer01 = new Affine(Params[0], Params[1]);
layers.Add("A1", affineLayer01);
dropout01 = new Dropout(0.15);
layers.Add("D1", dropout01);
///第一次非线性激活
layers.Add("ReLU1", new ReLU());
///第二次线性组合
affineLayer02 = new Affine(Params[2], Params[3]);
layers.Add("A2", affineLayer02);
dropout02 = new Dropout(0.15);
layers.Add("D2", dropout02);
///输出层 非线性激活 输出
softmaxWithLoss = new SoftmaxWithLoss();
}
public void LayerTypes()
{
_ = new Dense(3).Type();
_ = new Dropout(3, 0.5f).Type();
_ = new Output(3).Type();
_ = new Result(3).Type();
}
public void TestToJson()
{
var mp = new MaxPooling1D(poolSize: 2, padding: "same");
var jobj = mp.ToJObject();
var dropout = new Dropout(0.3, seed: 3984);
jobj = dropout.ToJObject();
}
public Dropout(Dropout dropout)
{
this.random = RandomProvider.GetRandom();
this.rate = dropout.rate;
this.maskDictionary = new Dictionary <int, double>();
foreach (var keyValuePair in dropout.maskDictionary)
{
this.maskDictionary.Add(keyValuePair.Key, keyValuePair.Value);
}
}
public void OutputShapeIsEqualToInputLayerShape()
{
var context = new ModelCompilationContext(new TFGraph());
var input = new Input(new [] { 10L });
var layer = new Dropout(0.2, input);
layer.Compile(context);
layer.OutputShape.Should().BeEquivalentTo(input.OutputShape);
}
#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format
public MultiHeadAttention(
int embeddingDim,
int numHeads,
int?kDim = null,
int?vDim = null,
double dropout = 0.0,
bool bias = true,
bool addBiasKv = false,
bool addZeroAttention = false,
bool selfAttention = false,
bool encoderDecoderAttention = false)
: base(nameof(MultiHeadAttention))
{
_embeddingDim = embeddingDim;
_kDim = kDim ?? embeddingDim;
_vDim = vDim ?? embeddingDim;
_qkvSameDim = (_kDim == _embeddingDim) && (_vDim == _embeddingDim);
_numHeads = numHeads;
_dropout = dropout;
_headDim = _embeddingDim / _numHeads;
_scaling = Math.Pow(_headDim, -0.5);
if (_headDim * _numHeads != _embeddingDim)
{
throw new ArgumentException("EmbeddingDim must be divisible by NumHeads");
}
_selfAttention = selfAttention;
_encoderDecoderAttention = encoderDecoderAttention;
if (_selfAttention && !_qkvSameDim)
{
throw new ArgumentException("Self-attention requires query, key and value to be of the same size");
}
_addBiasProj = bias;
_addBiasKv = addBiasKv;
_addZeroAttention = addZeroAttention;
QProjection = torch.nn.Linear(_embeddingDim, _embeddingDim, _addBiasProj);
KProjection = torch.nn.Linear(_kDim, _embeddingDim, _addBiasProj);
VProjection = torch.nn.Linear(_vDim, _embeddingDim, _addBiasProj);
if (_addBiasKv)
{
KBias = torch.zeros(1, 1, _embeddingDim).AsParameter();
VBias = torch.zeros(1, 1, _embeddingDim).AsParameter();
}
OutProjLinear = torch.nn.Linear(_embeddingDim, _embeddingDim, _addBiasProj);
DropoutLayer = torch.nn.Dropout(_dropout);
Initialize();
RegisterComponents();
}
public Sequential CreateSequential(JToken model)
{
var seq = new Sequential(controller);
var layers = model.SelectToken("config").Children();
foreach (var layer in layers)
{
var layerType = layer.SelectToken("class_name");
switch (layerType.Value <String>())
{
case "Linear":
// weight float tensor
var weightData = layer.SelectToken("config.weights.data").ToObject <float[]>();
var input = layer.SelectToken("config.input").ToObject <int>();
var output = layer.SelectToken("config.output").ToObject <int>();
float[] biasData = null;
if (layer.SelectToken("config.bias") == null)
{
biasData = layer.SelectToken("config.bias.data").ToObject <float[]>();
}
Linear linear = new Linear(controller, input: input, output: output, weights: weightData, bias: biasData);
seq.AddLayer(linear);
break;
case "ReLU":
seq.AddLayer(new ReLU(controller));
break;
case "Log":
seq.AddLayer(new OpenMined.Syft.Layer.Log(controller));
break;
case "Dropout":
var rate = layer.SelectToken("config.rate").ToObject <float>();
var dropout = new Dropout(controller, rate);
seq.AddLayer(dropout);
break;
case "Softmax":
var dim = layer.SelectToken("config.dim").ToObject <int>();
seq.AddLayer(new Softmax(controller, dim));
break;
case "Sigmoid":
seq.AddLayer(new Sigmoid(controller));
break;
}
}
return(seq);
}
private Sequential CreateSequential(JToken model)
{
var seq = new Sequential(controller);
var layers = model.SelectToken("config").Children();
foreach (var layer in layers)
{
var layerType = layer.SelectToken("class_name");
switch (layerType.Value <String>())
{
case "Linear":
// weight float tensor
var weightData = layer.SelectToken("config.weights.data").ToObject <float[]>();
var weightShape = layer.SelectToken("config.weights.shape").ToObject <int[]>();
var weightTensor = controller.floatTensorFactory.Create(_data: weightData, _shape: weightShape, _autograd: true);
// bias float tensor
var biasData = layer.SelectToken("config.bias.data").ToObject <float[]>();
var biasShape = layer.SelectToken("config.bias.shape").ToObject <int[]>();
var biasTensor = controller.floatTensorFactory.Create(_data: biasData, _shape: biasShape, _autograd: true);
var input = layer.SelectToken("config.input").ToObject <int>();
var output = layer.SelectToken("config.output").ToObject <int>();
var linear = new Linear(controller, input: input, output: output, weights: weightTensor, bias: biasTensor);
seq.AddLayer(linear);
break;
case "ReLU":
seq.AddLayer(new ReLU(controller));
break;
case "Log":
seq.AddLayer(new OpenMined.Syft.Layer.Log(controller));
break;
case "Dropout":
var rate = layer.SelectToken("config.rate").ToObject <float>();
var dropout = new Dropout(controller, rate);
seq.AddLayer(dropout);
break;
case "Softmax":
var dim = layer.SelectToken("config.dim").ToObject <int>();
seq.AddLayer(new Softmax(controller, dim));
break;
}
}
return(seq);
}
public ILayer CreateProduct(IKernelDescriptor descriptor)
{
if (descriptor is Dropout)
{
Dropout dropout = descriptor as Dropout;
ILayer layer = new DropoutLayer(dropout.Rate, dropout.NoiseShape);
return(layer);
}
return(null);
}
public PositionalEncoding(long dmodel, double dropout, int maxLen = 5000) : base("PositionalEncoding")
{
this.dropout = Dropout(dropout);
var pe = Float32Tensor.zeros(new long[] { maxLen, dmodel });
var position = Float32Tensor.arange(0, maxLen, 1).unsqueeze(1);
var divTerm = (Float32Tensor.arange(0, dmodel, 2) * (-Math.Log(10000.0) / dmodel)).exp();
pe[TorchTensorIndex.Ellipsis, TorchTensorIndex.Slice(0, null, 2)] = (position * divTerm).sin();
pe[TorchTensorIndex.Ellipsis, TorchTensorIndex.Slice(1, null, 2)] = (position * divTerm).cos();
this.pe = pe.unsqueeze(0).transpose(0, 1);
RegisterComponents();
}
public static void Run()
{
sw = new Stopwatch();
Console.WriteLine("Generating Test Data...");
NdArray input = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
NdArray inputImage = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
Console.WriteLine("Generated Test Data");
Console.WriteLine("Init Linear");
Linear linear = new Linear(INPUT_SIZE, OUTPUT_SIZE);
Console.WriteLine("Init Tanh");
Tanh tanh = new Tanh();
Console.WriteLine("Init Sigmoid");
Sigmoid sigmoid = new Sigmoid();
Console.WriteLine("Init ReLU");
ReLU relu = new ReLU();
Console.WriteLine("Init LeakyReLU");
LeakyReLU leakyRelu = new LeakyReLU();
Console.WriteLine("Init MaxPooling");
MaxPooling maxPooling = new MaxPooling(2);
Console.WriteLine("Init Convolution2D");
Convolution2D conv2d = new Convolution2D(3, 32, 3);
Console.WriteLine("Init Deconvolution2D");
Deconvolution2D deconv2d = new Deconvolution2D(32, 3, 3);
Dropout dropout = new Dropout();
TestLayer(linear, input);
Console.WriteLine("aaaaaaaaaaaa");
Console.ReadLine();
TestLayer(tanh, input);
TestLayer(sigmoid, input);
TestLayer(relu, input);
TestLayer(leakyRelu, input);
TestLayer(maxPooling, inputImage);
TestLayer(conv2d, inputImage);
TestLayer(deconv2d, inputImage);
TestLayer(dropout, input);
}
public void CreatesLayerConfigurationDuringCompilation()
{
var context = new ModelCompilationContext(new TFGraph());
var input = new Input(new [] { 10L });
var layer = new Dropout(0.2, input);
layer.Compile(context);
layer.Configuration.Should().NotBeNull();
layer.Configuration.Parameters.Count().Should().Be(0);
layer.Configuration.Initializers.Count().Should().Be(0);
layer.Configuration.Output.Should().NotBeNull();
}
public Illustration2Vec()
{
var inputTensor = new InputLayer(new int?[] { 224, 224, 3 });
var vgg16_model = VGG16Model.CreateModel("");
foreach (var layer in vgg16_model.layers.Take(12))
{
layer.trainable = false;
}
var x = vgg16_model.layers.Last().output;
// List <KerasSharp.Engine.Topology.Tensor> x = null;
x = new Flatten().Call(x);
x = new BatchNormalization().Call(x);
x = new Dense(5000, activation: "relu").Call(x);
x = new Dropout(0.3).Call(x);
x = new Dense(5000, activation: "sigmoid").Call(x);
model = new Model(vgg16_model.input, x, "altI2v");
model.Compile(new Adam(), new BinaryCrossEntropy());
}
#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format
public SelfAttentionLayer(
int embeddingDim = 768,
int numAttentionHeads = 8,
double dropoutRate = 0.1f,
double attentionDropoutRate = 0.1f,
bool addBiasKv = false,
bool addZeroAttention = false)
: base(nameof(SelfAttentionLayer))
{
SelfAttention = new MultiHeadAttention(
embeddingDim,
numAttentionHeads,
dropout: attentionDropoutRate,
addBiasKv: addBiasKv,
addZeroAttention: addZeroAttention,
selfAttention: true);
DropoutLayer = torch.nn.Dropout(dropoutRate);
// Layer norm associated with the self attention layer
LayerNorm = torch.nn.LayerNorm(new long[] { embeddingDim });
RegisterComponents();
}
private Sequential CreateSequential(List <String> model)
{
// TODO just assumes it is all in a seq model the seq model should probably
// be in the JSON????
var seq = new Sequential(controller);
foreach (var l in model)
{
var config = JObject.Parse(l);
Layer layer = null;
switch ((string)config["name"])
{
case "linear":
layer = new Linear(controller, (int)config["input"], (int)config["output"]);
break;
case "softmax":
layer = new Softmax(controller, (int)config["dim"]);
break;
case "relu":
layer = new ReLU(controller);
break;
case "log":
layer = new Log(controller);
break;
case "dropout":
layer = new Dropout(controller, (float)config["rate"]);
break;
}
seq.AddLayer(layer);
}
return(seq);
}
public TransformerEncoder(
int paddingIdx,
int vocabSize,
double dropout = 0.1f,
double attentionDropout = 0.1f,
double activationDropout = 0.1f,
string activationFn = "relu",
bool dynamicDropout = false,
bool addBiasKv = false,
bool addZeroAttention = false,
int maxSeqLen = 256,
bool learnedPositionEmbedding = true,
int embedSize = -1,
int?embedScale = null,
IList <int> arches = null,
bool usePositionEmbedding = true,
bool offsetPositionsByPadding = true,
int numSegments = 2,
bool encoderNormalizeBefore = false,
int numEncoderLayers = 6,
bool applyBertInit = false,
bool freezeEmbeddings = false,
bool freezeLayers = false,
bool freezeTransfer = false,
int nTransLayersToFreeze = 0)
: base(nameof(TransformerEncoder))
{
Contracts.AssertValue(arches);
Contracts.AssertNonEmpty(arches);
PaddingIdx = paddingIdx;
DiscreteArches = arches.ToList();
DistillBlocks = 4;
// Embedding modules
EmbedScale = embedScale;
TokenEmbedding = torch.nn.Embedding(vocabSize, embedSize, paddingIdx);
PositionalEmbedding = usePositionEmbedding
? PositionalEmbedding.GetPositionalEmbedding(maxSeqLen, embedSize,
paddingIdx, learnedPositionEmbedding)
: null;
SegmentEmbedding = numSegments > 0
? torch.nn.Embedding(numSegments, embedSize)
: null;
EmbeddingLayerNorm = encoderNormalizeBefore
? torch.nn.LayerNorm(new long[] { embedSize })
: null;
DropoutLayer = torch.nn.Dropout(dropout);
ModelUtils.InitNormal(TokenEmbedding.weight, mean: 0.0, std: 0.02);
ModelUtils.InitZeros(TokenEmbedding.weight[paddingIdx]);
if (SegmentEmbedding != null)
{
ModelUtils.InitNormal(SegmentEmbedding.weight, mean: 0.0, std: 0.02);
}
// Encoder layers
var layers = Enumerable.Range(0, numEncoderLayers)
.Select(i => new TransformerCellDiscrete(
arches[i],
dropout,
attentionDropout,
activationDropout,
activationFn,
addBiasKv,
addZeroAttention,
dynamicDropout) as torch.nn.Module)
.ToArray();
Layers = new ModuleList(layers);
var blockPerLayer = numEncoderLayers / DistillBlocks;
HiddenSizePerBlock = CheckBlockHiddenSize(blockPerLayer);
EmbedTransfer = new EmbedTransferDiscrete(embedSize, HiddenSizePerBlock[0]);
var hiddenSizePerBlockExtend = HiddenSizePerBlock.Append(HiddenSizePerBlock[HiddenSizePerBlock.Count - 1]).ToList();
var hiddenTransferList = Enumerable.Range(0, HiddenSizePerBlock.Count)
.Select(i => new HiddenTransferDiscrete(hiddenSizePerBlockExtend[i],
hiddenSizePerBlockExtend[i + 1]) as torch.nn.Module)
.ToArray();
HiddenTransferList = new ModuleList(hiddenTransferList);
if (freezeEmbeddings)
{
ModelUtils.FreezeModuleParams(TokenEmbedding);
ModelUtils.FreezeModuleParams(PositionalEmbedding);
ModelUtils.FreezeModuleParams(SegmentEmbedding);
ModelUtils.FreezeModuleParams(EmbeddingLayerNorm);
}
if (freezeLayers)
{
ModelUtils.FreezeModuleParams(Layers);
ModelUtils.FreezeModuleParams(HiddenTransferList);
}
if (freezeTransfer)
{
ModelUtils.FreezeModuleParams(HiddenTransferList);
}
for (var i = 0; i < nTransLayersToFreeze; ++i)
{
ModelUtils.FreezeModuleParams(Layers[i]);
}
RegisterComponents();
}
public static void Run()
{
Stopwatch sw = new Stopwatch();
NdArray inputArrayCpu = new NdArray(Initializer.GetRealArray(INPUT_SIZE));
NdArray inputArrayGpu = new NdArray(Initializer.GetRealArray(INPUT_SIZE));
//Linear
Linear linear = new Linear(INPUT_SIZE, OUTPUT_SIZE);
Console.WriteLine("◆" + linear.Name);
sw.Restart();
NdArray[] gradArrayCpu = linear.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data; //DataをGradとして使用
sw.Restart();
linear.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (linear.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = linear.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
linear.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
//Tanh
TanhActivation tanh = new TanhActivation();
Console.WriteLine("\n◆" + tanh.Name);
sw.Restart();
gradArrayCpu = tanh.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
tanh.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (tanh.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = tanh.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
tanh.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
//Sigmoid
Sigmoid sigmoid = new Sigmoid();
Console.WriteLine("\n◆" + sigmoid.Name);
sw.Restart();
gradArrayCpu = sigmoid.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sigmoid.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (sigmoid.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = sigmoid.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
sigmoid.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
//ReLU
ReLU relu = new ReLU();
Console.WriteLine("\n◆" + relu.Name);
sw.Restart();
gradArrayCpu = relu.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
relu.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (relu.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = relu.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
relu.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
//LeakyReLU
LeakyReLU leakyRelu = new LeakyReLU();
Console.WriteLine("\n◆" + leakyRelu.Name);
sw.Restart();
gradArrayCpu = leakyRelu.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
leakyRelu.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (leakyRelu.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = leakyRelu.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
leakyRelu.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
NdArray inputImageArrayGpu = new NdArray(Initializer.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
NdArray inputImageArrayCpu = new NdArray(Initializer.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
//MaxPooling
MaxPooling2D maxPooling2D = new MaxPooling2D(3);
Console.WriteLine("\n◆" + maxPooling2D.Name);
sw.Restart();
NdArray[] gradImageArrayCpu = maxPooling2D.Forward(inputImageArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
maxPooling2D.Backward(gradImageArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (maxPooling2D.SetGpuEnable(true))
{
sw.Restart();
maxPooling2D.Forward(inputImageArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
//メモリ転送のみのため実装がない
Console.WriteLine("Backward[Gpu] : None");
}
//Conv2D
Convolution2D conv2d = new Convolution2D(3, 3, 3);
Console.WriteLine("\n◆" + conv2d.Name);
sw.Restart();
gradImageArrayCpu = conv2d.Forward(inputImageArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
conv2d.Backward(gradImageArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (conv2d.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradImageArrayGpu = conv2d.Forward(inputImageArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data;
sw.Restart();
conv2d.Backward(gradImageArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
//Deconv2D
Deconvolution2D deconv2d = new Deconvolution2D(3, 3, 3);
Console.WriteLine("\n◆" + deconv2d.Name);
sw.Restart();
gradImageArrayCpu = deconv2d.Forward(inputImageArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
deconv2d.Backward(gradImageArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (deconv2d.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradImageArrayGpu = deconv2d.Forward(inputImageArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data;
sw.Restart();
deconv2d.Backward(gradImageArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
//Dropout
Dropout dropout = new Dropout();
Console.WriteLine("\n◆" + dropout.Name);
sw.Restart();
gradArrayCpu = dropout.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
dropout.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (dropout.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = dropout.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
dropout.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
}
public void ProcessMessage(string json_message, MonoBehaviour owner, Action <string> response)
{
Command msgObj = JsonUtility.FromJson <Command> (json_message);
try
{
switch (msgObj.objectType)
{
case "Optimizer":
{
if (msgObj.functionCall == "create")
{
string optimizer_type = msgObj.tensorIndexParams[0];
// Extract parameters
List <int> p = new List <int>();
for (int i = 1; i < msgObj.tensorIndexParams.Length; i++)
{
p.Add(int.Parse(msgObj.tensorIndexParams[i]));
}
List <float> hp = new List <float>();
for (int i = 0; i < msgObj.hyperParams.Length; i++)
{
hp.Add(float.Parse(msgObj.hyperParams[i]));
}
Optimizer optim = null;
if (optimizer_type == "sgd")
{
optim = new SGD(this, p, hp[0], hp[1], hp[2]);
}
else if (optimizer_type == "rmsprop")
{
optim = new RMSProp(this, p, hp[0], hp[1], hp[2], hp[3]);
}
else if (optimizer_type == "adam")
{
optim = new Adam(this, p, hp[0], hp[1], hp[2], hp[3], hp[4]);
}
response(optim.Id.ToString());
return;
}
else
{
Optimizer optim = this.GetOptimizer(msgObj.objectIndex);
response(optim.ProcessMessage(msgObj, this));
return;
}
}
case "FloatTensor":
{
if (msgObj.objectIndex == 0 && msgObj.functionCall == "create")
{
FloatTensor tensor = floatTensorFactory.Create(_shape: msgObj.shape, _data: msgObj.data, _shader: this.Shader);
response(tensor.Id.ToString());
return;
}
else
{
FloatTensor tensor = floatTensorFactory.Get(msgObj.objectIndex);
// Process message's function
response(tensor.ProcessMessage(msgObj, this));
return;
}
}
case "IntTensor":
{
if (msgObj.objectIndex == 0 && msgObj.functionCall == "create")
{
int[] data = new int[msgObj.data.Length];
for (int i = 0; i < msgObj.data.Length; i++)
{
data[i] = (int)msgObj.data[i];
}
IntTensor tensor = intTensorFactory.Create(_shape: msgObj.shape, _data: data);
response(tensor.Id.ToString());
return;
}
else
{
IntTensor tensor = intTensorFactory.Get(msgObj.objectIndex);
// Process message's function
response(tensor.ProcessMessage(msgObj, this));
return;
}
}
case "agent":
{
if (msgObj.functionCall == "create")
{
Layer model = (Layer)GetModel(int.Parse(msgObj.tensorIndexParams[0]));
Optimizer optimizer = optimizers[int.Parse(msgObj.tensorIndexParams[1])];
response(new Syft.NN.RL.Agent(this, model, optimizer).Id.ToString());
return;
}
//Debug.Log("Getting Model:" + msgObj.objectIndex);
Syft.NN.RL.Agent agent = this.GetAgent(msgObj.objectIndex);
response(agent.ProcessMessageLocal(msgObj, this));
return;
}
case "model":
{
if (msgObj.functionCall == "create")
{
string model_type = msgObj.tensorIndexParams[0];
Debug.LogFormat("<color=magenta>createModel:</color> {0}", model_type);
if (model_type == "linear")
{
response(this.BuildLinear(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "relu")
{
response(this.BuildReLU().Id.ToString());
return;
}
else if (model_type == "log")
{
response(this.BuildLog().Id.ToString());
return;
}
else if (model_type == "dropout")
{
response(this.BuildDropout(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "sigmoid")
{
response(this.BuildSigmoid().Id.ToString());
return;
}
else if (model_type == "sequential")
{
response(this.BuildSequential().Id.ToString());
return;
}
else if (model_type == "softmax")
{
response(this.BuildSoftmax(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "logsoftmax")
{
response(this.BuildLogSoftmax(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "tanh")
{
response(new Tanh(this).Id.ToString());
return;
}
else if (model_type == "crossentropyloss")
{
response(new CrossEntropyLoss(this, int.Parse(msgObj.tensorIndexParams[1])).Id.ToString());
return;
}
else if (model_type == "categorical_crossentropy")
{
response(new CategoricalCrossEntropyLoss(this).Id.ToString());
return;
}
else if (model_type == "nllloss")
{
response(new NLLLoss(this).Id.ToString());
return;
}
else if (model_type == "mseloss")
{
response(new MSELoss(this).Id.ToString());
return;
}
else if (model_type == "embedding")
{
response(new Embedding(this, int.Parse(msgObj.tensorIndexParams[1]), int.Parse(msgObj.tensorIndexParams[2])).Id.ToString());
return;
}
else
{
Debug.LogFormat("<color=red>Model Type Not Found:</color> {0}", model_type);
}
}
else
{
//Debug.Log("Getting Model:" + msgObj.objectIndex);
Model model = this.GetModel(msgObj.objectIndex);
response(model.ProcessMessage(msgObj, this));
return;
}
response("Unity Error: SyftController.processMessage: Command not found:" + msgObj.objectType + ":" + msgObj.functionCall);
return;
}
case "controller":
{
if (msgObj.functionCall == "num_tensors")
{
response(floatTensorFactory.Count() + "");
return;
}
else if (msgObj.functionCall == "num_models")
{
response(models.Count + "");
return;
}
else if (msgObj.functionCall == "new_tensors_allowed")
{
Debug.LogFormat("New Tensors Allowed:{0}", msgObj.tensorIndexParams[0]);
if (msgObj.tensorIndexParams[0] == "True")
{
allow_new_tensors = true;
}
else if (msgObj.tensorIndexParams[0] == "False")
{
allow_new_tensors = false;
}
else
{
throw new Exception("Invalid parameter for new_tensors_allowed. Did you mean true or false?");
}
response(allow_new_tensors + "");
return;
}
else if (msgObj.functionCall == "load_floattensor")
{
FloatTensor tensor = floatTensorFactory.Create(filepath: msgObj.tensorIndexParams[0], _shader: this.Shader);
response(tensor.Id.ToString());
return;
}
else if (msgObj.functionCall == "set_seed")
{
Random.InitState(int.Parse(msgObj.tensorIndexParams[0]));
response("Random seed set!");
return;
}
else if (msgObj.functionCall == "concatenate")
{
List <int> tensor_ids = new List <int>();
for (int i = 1; i < msgObj.tensorIndexParams.Length; i++)
{
tensor_ids.Add(int.Parse(msgObj.tensorIndexParams[i]));
}
FloatTensor result = Functional.Concatenate(floatTensorFactory, tensor_ids, int.Parse(msgObj.tensorIndexParams[0]));
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "ones")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Ones(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "randn")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Randn(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "random")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Random(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "zeros")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Zeros(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "model_from_json")
{
Debug.Log("Loading Model from JSON:");
var json_str = msgObj.tensorIndexParams[0];
var config = JObject.Parse(json_str);
Sequential model;
if ((string)config["class_name"] == "Sequential")
{
model = this.BuildSequential();
}
else
{
response("Unity Error: SyftController.processMessage: while Loading model, Class :" + config["class_name"] + " is not implemented");
return;
}
for (int i = 0; i < config["config"].ToList().Count; i++)
{
var layer_desc = config["config"][i];
var layer_config_desc = layer_desc["config"];
if ((string)layer_desc["class_name"] == "Linear")
{
int previous_output_dim;
if (i == 0)
{
previous_output_dim = (int)layer_config_desc["batch_input_shape"][layer_config_desc["batch_input_shape"].ToList().Count - 1];
}
else
{
previous_output_dim = (int)layer_config_desc["units"];
}
string[] parameters = { "linear", previous_output_dim.ToString(), layer_config_desc["units"].ToString(), "Xavier" };
Layer layer = this.BuildLinear(parameters);
model.AddLayer(layer);
string activation_name = layer_config_desc["activation"].ToString();
if (activation_name != "linear")
{
Layer activation;
if (activation_name == "softmax")
{
parameters = new string[] { activation_name, "1" };
activation = this.BuildSoftmax(parameters);
}
else if (activation_name == "relu")
{
activation = this.BuildReLU();
}
else
{
response("Unity Error: SyftController.processMessage: while Loading activations, Activation :" + activation_name + " is not implemented");
return;
}
model.AddLayer(activation);
}
}
else
{
response("Unity Error: SyftController.processMessage: while Loading layers, Layer :" + layer_desc["class_name"] + " is not implemented");
return;
}
}
response(model.Id.ToString());
return;
}
else if (msgObj.functionCall == "from_proto")
{
Debug.Log("Loading Model from ONNX:");
var filename = msgObj.tensorIndexParams[0];
var input = File.OpenRead(filename);
ModelProto modelProto = ModelProto.Parser.ParseFrom(input);
Sequential model = this.BuildSequential();
foreach (NodeProto node in modelProto.Graph.Node)
{
Layer layer;
GraphProto g = ONNXTools.GetSubGraphFromNodeAndMainGraph(node, modelProto.Graph);
if (node.OpType == "Gemm")
{
layer = new Linear(this, g);
}
else if (node.OpType == "Dropout")
{
layer = new Dropout(this, g);
}
else if (node.OpType == "Relu")
{
layer = new ReLU(this, g);
}
else if (node.OpType == "Softmax")
{
layer = new Softmax(this, g);
}
else
{
response("Unity Error: SyftController.processMessage: Layer not yet implemented for deserialization:");
return;
}
model.AddLayer(layer);
}
response(model.Id.ToString());
return;
}
else if (msgObj.functionCall == "to_proto")
{
ModelProto model = this.ToProto(msgObj.tensorIndexParams);
string filename = msgObj.tensorIndexParams[2];
string type = msgObj.tensorIndexParams[3];
if (type == "json")
{
response(model.ToString());
}
else
{
using (var output = File.Create(filename))
{
model.WriteTo(output);
}
response(new FileInfo(filename).FullName);
}
return;
}
response("Unity Error: SyftController.processMessage: Command not found:" + msgObj.objectType + ":" + msgObj.functionCall);
return;
}
case "Grid":
if (msgObj.functionCall == "learn")
{
var inputId = int.Parse(msgObj.tensorIndexParams[0]);
var targetId = int.Parse(msgObj.tensorIndexParams[1]);
response(this.grid.Run(inputId, targetId, msgObj.configurations, owner));
return;
}
if (msgObj.functionCall == "getResults")
{
this.grid.GetResults(msgObj.experimentId, response);
return;
}
// like getResults but doesn't pause to wait for results
// this function will return right away telling you if
// it knows whether or not it is done
if (msgObj.functionCall == "checkStatus")
{
this.grid.CheckStatus(msgObj.experimentId, response);
return;
}
break;
default:
break;
}
}
catch (Exception e)
{
Debug.LogFormat("<color=red>{0}</color>", e.ToString());
response("Unity Error: " + e.ToString());
return;
}
// If not executing createTensor or tensor function, return default error.
response("Unity Error: SyftController.processMessage: Command not found:" + msgObj.objectType + ":" + msgObj.functionCall);
return;
}
public Model(Context ctx, Config cfg, bool isTraining = true, bool usingCuDnn = true)
{
Config = cfg;
IsTraining = isTraining;
UsingCuDnn = usingCuDnn;
Inputs = Variable <int>(PartialShape.Create(cfg.NumSteps, cfg.BatchSize));
Targets = Variable <int>(PartialShape.Create(cfg.NumSteps, cfg.BatchSize));
// embedding
Embedding = new Embedding <float>(Inputs, cfg.VocabSize, cfg.HiddenSize, initScale: cfg.InitScale);
// add dropout
EmbeddedOutput = Embedding.Output;
if (isTraining && cfg.KeepProb < 1.0)
{
var dropout = new Dropout <float>(EmbeddedOutput, dropoutProb: 1.0 - cfg.KeepProb);
EmbeddedOutput = dropout.Output;
}
// rnn layer, dropout for intermediate lstm layers and for output
if (usingCuDnn)
{
RnnAccelerated = new Rnn <float>(new LstmRnnType(forgetBiasInit: 0.0), EmbeddedOutput, cfg.NumLayers, cfg.HiddenSize, isTraining: isTraining, dropout: isTraining && cfg.KeepProb < 1.0 ? 1.0 - Config.KeepProb : 0.0);
RnnOutput = RnnAccelerated.Y;
if (isTraining && cfg.KeepProb < 1.0)
{
var dropout = new Dropout <float>(RnnOutput, dropoutProb: 1.0 - cfg.KeepProb);
RnnOutput = dropout.Output;
}
}
else
{
RnnDirect = new Lstm <float> [cfg.NumLayers];
for (var i = 0; i < cfg.NumLayers; ++i)
{
var lstm = new Lstm <float>(i == 0 ? EmbeddedOutput : RnnOutput, cfg.HiddenSize, forgetBiasInit: 0.0);
RnnDirect[i] = lstm;
RnnOutput = lstm.Y;
if (isTraining && cfg.KeepProb < 1.0)
{
var dropout = new Dropout <float>(RnnOutput, dropoutProb: 1.0 - cfg.KeepProb);
RnnOutput = dropout.Output;
}
}
}
FC = new FullyConnected <float>(RnnOutput.Reshape(RnnOutput.Shape[0] * RnnOutput.Shape[1], RnnOutput.Shape[2]), cfg.VocabSize);
Loss = new SoftmaxCrossEntropySparse <float>(FC.Output, Targets.Reshape(Targets.Shape[0] * Targets.Shape[1]));
Optimizer = new GradientDescentOptimizer(ctx, Loss.Loss, cfg.LearningRate, new GlobalNormGradientClipper(cfg.MaxGradNorm));
// warmup to force JIT compilation to get timings without JIT overhead
Optimizer.Initalize();
ResetStates();
Optimizer.AssignTensor(Inputs, Fill(Shape.Create(Inputs.Shape.AsArray), 0));
Optimizer.AssignTensor(Targets, Fill(Shape.Create(Targets.Shape.AsArray), 0));
Optimizer.Forward();
if (isTraining)
{
Optimizer.Backward();
}
// now reset states
Optimizer.Initalize();
ResetStates();
}
public bool CreateLayer(int nCount, ELayerType type, ActivationSettings activationSettings)
{
Layer.Utility.Layer layer;
switch (type)
{
case ELayerType.Invalid:
throw new ArgumentException("Invalid \"type\" argument.");
case ELayerType.AveragePooling:
layer = new AveragePooling(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.AverageUnpooling:
layer = new AverageUnpooling(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.Convolutional:
layer = new Convolutional(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.Deconvolutional:
layer = new Deconvolutional(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.Dropout:
layer = new Dropout(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.FullyConnected:
layer = new FullyConnected(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.GatedRecurrent:
layer = new GatedRecurrent(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.LSTM:
layer = new LSTM(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.MaxPooling:
layer = new MaxPooling(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.MaxUnpooling:
layer = new MaxUnpooling(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
case ELayerType.Recurrent:
layer = new Recurrent(nCount, Layers.Count, activationSettings);
Layers.Add(layer);
return(true);
default:
throw new ArgumentException("Invalid \"type\" argument.");
}
}
public static void Run()
{
Stopwatch sw = new Stopwatch();
NdArray inputArrayCpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
NdArray inputArrayGpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
NdArray[] gradArrayCpu = null;
//Linear
Linear linear = new Linear(INPUT_SIZE, OUTPUT_SIZE);
Console.WriteLine("◆" + linear.Name);
if (TestCpu)
{
sw.Restart();
gradArrayCpu = linear.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data; //Use Data as Grad
sw.Restart();
linear.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
if (linear.SetGpuEnable(true))
{
NdArray[] gradArrayGpu = null;
while (true)
{
sw.Restart();
gradArrayGpu = linear.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
linear.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
//Tanh
Tanh tanh = new Tanh();
Console.WriteLine("\n ◆" + tanh.Name);
sw.Restart();
gradArrayCpu = tanh.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
tanh.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (tanh.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = tanh.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
tanh.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
//Sigmoid
Sigmoid sigmoid = new Sigmoid();
Console.WriteLine("\n ◆" + sigmoid.Name);
sw.Restart();
gradArrayCpu = sigmoid.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sigmoid.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (sigmoid.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = sigmoid.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
sigmoid.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
//ReLU
ReLU relu = new ReLU();
Console.WriteLine("\n ◆" + relu.Name);
sw.Restart();
gradArrayCpu = relu.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
relu.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (relu.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = relu.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
relu.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
//LeakyReLU
LeakyReLU leakyRelu = new LeakyReLU();
Console.WriteLine("\n ◆" + leakyRelu.Name);
sw.Restart();
gradArrayCpu = leakyRelu.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
leakyRelu.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (leakyRelu.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = leakyRelu.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
leakyRelu.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
NdArray inputImageArrayGpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
NdArray inputImageArrayCpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
//MaxPooling
MaxPooling maxPooling = new MaxPooling(3);
Console.WriteLine("\n ◆" + maxPooling.Name);
sw.Restart();
NdArray[] gradImageArrayCpu = maxPooling.Forward(inputImageArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
maxPooling.Backward(gradImageArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (maxPooling.SetGpuEnable(true))
{
sw.Restart();
maxPooling.Forward(inputImageArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
//There is no implementation for memory transfer only
Console.WriteLine("Backward[Gpu] : None");
}
//Conv2D
Convolution2D conv2d = new Convolution2D(3, 3, 3);
Console.WriteLine("\n ◆" + conv2d.Name);
sw.Restart();
gradImageArrayCpu = conv2d.Forward(inputImageArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
conv2d.Backward(gradImageArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (conv2d.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradImageArrayGpu = conv2d.Forward(inputImageArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data;
sw.Restart();
conv2d.Backward(gradImageArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
//Deconv2D
Deconvolution2D deconv2d = new Deconvolution2D(3, 3, 3);
Console.WriteLine("\n ◆" + deconv2d.Name);
sw.Restart();
gradImageArrayCpu = deconv2d.Forward(inputImageArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
deconv2d.Backward(gradImageArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (deconv2d.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradImageArrayGpu = deconv2d.Forward(inputImageArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradImageArrayGpu[0].Grad = gradImageArrayGpu[0].Data;
sw.Restart();
deconv2d.Backward(gradImageArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
//Dropout
Dropout dropout = new Dropout();
Console.WriteLine("\n ◆" + dropout.Name);
sw.Restart();
gradArrayCpu = dropout.Forward(inputArrayCpu);
sw.Stop();
Console.WriteLine("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
dropout.Backward(gradArrayCpu);
sw.Stop();
Console.WriteLine("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
if (dropout.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = dropout.Forward(inputArrayGpu);
sw.Stop();
Console.WriteLine("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
dropout.Backward(gradArrayGpu);
sw.Stop();
Console.WriteLine("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μs");
}
}
private List <IKernelDescriptor> ReadDescriptors(JObject model)
{
List <IKernelDescriptor> dscps = model.SelectToken("descriptors").Select(layer => {
IKernelDescriptor descriptor = null;
String layerName = (String)layer.SelectToken("layer");
switch (layerName)
{
case "AvgPooling1D":
descriptor = new AvgPooling1D(
(int)layer.SelectToken("padding"),
(int)layer.SelectToken("stride"),
(int)layer.SelectToken("kernel_size"));
break;
case "GlobalAveragePooling1D":
descriptor = new GlobalAvgPooling1D();
break;
case "AvgPooling2D":
descriptor = new AvgPooling2D((int)layer.SelectToken("padding_vl"), (int)layer.SelectToken("padding_hz"),
(int)layer.SelectToken("stride_vl"), (int)layer.SelectToken("stride_hz"),
(int)layer.SelectToken("kernel_height"), (int)layer.SelectToken("kernel_width"));
break;
case "GlobalAveragePooling2D":
descriptor = new GlobalAvgPooling2D();
break;
case "BatchNormalization":
descriptor = new BatchNormalization(
(int)layer.SelectToken("epsilon"));
break;
case "Cropping1D":
descriptor = new Cropping1D(
(int)layer.SelectToken("trimBegin"),
(int)layer.SelectToken("trimEnd"));
break;
case "Cropping2D":
descriptor = new Cropping2D(
(int)layer.SelectToken("topTrim"),
(int)layer.SelectToken("bottomTrim"),
(int)layer.SelectToken("leftTrim"),
(int)layer.SelectToken("rightTrim"));
break;
case "MaxPooling1D":
descriptor = new MaxPooling1D(
(int)layer.SelectToken("padding"),
(int)layer.SelectToken("stride"),
(int)layer.SelectToken("kernel_size"));
break;
case "GlobalMaxPooling1D":
descriptor = new GlobalMaxPooling1D();
break;
case "MaxPooling2D":
descriptor = new MaxPooling2D((int)layer.SelectToken("padding_vl"), (int)layer.SelectToken("padding_hz"),
(int)layer.SelectToken("stride_vl"), (int)layer.SelectToken("stride_hz"),
(int)layer.SelectToken("kernel_height"), (int)layer.SelectToken("kernel_width"));
break;
case "GlobalMaxPooling2D":
descriptor = new GlobalMaxPooling2D();
break;
case "Convolution1D":
descriptor = new Convolution1D(
(int)layer.SelectToken("padding"),
(int)layer.SelectToken("stride"),
(int)layer.SelectToken("kernel_size"),
(int)layer.SelectToken("kernel_num"));
break;
case "Convolution2D":
descriptor = new Convolution2D((int)layer.SelectToken("padding_vl"), (int)layer.SelectToken("padding_hz"),
(int)layer.SelectToken("stride_vl"), (int)layer.SelectToken("stride_hz"),
(int)layer.SelectToken("kernel_height"), (int)layer.SelectToken("kernel_width"),
(int)layer.SelectToken("kernel_num"));
break;
case "Dense2D":
descriptor = new Dense2D((int)layer.SelectToken("units"));
break;
case "Dropout":
/*int h = (int)layer.SelectToken("height");
* int w = (int)layer.SelectToken("weight");
* int c = (int)layer.SelectToken("channel");
* int b = (int)layer.SelectToken("batch");*/
Data2D noiseShape = new Data2D(1, 2, 3, 2);
descriptor = new Dropout((double)layer.SelectToken("rate"), noiseShape);
break;
case "Input2D":
descriptor = new Input2D((int)layer.SelectToken("height"), (int)layer.SelectToken("width"),
(int)layer.SelectToken("channel"), (int)layer.SelectToken("batch"));
break;
case "Bias2D":
descriptor = new Bias2D();
break;
case "Permute":
descriptor = new Permute(
(int)layer.SelectToken("dim1"),
(int)layer.SelectToken("dim2"),
(int)layer.SelectToken("dim3"));
break;
case "Reshape":
descriptor = new Reshape2D(
(int)layer.SelectToken("height"),
(int)layer.SelectToken("width"),
(int)layer.SelectToken("channel"),
1);
break;
case "RepeatVector":
descriptor = new RepeatVector(
(int)layer.SelectToken("num"));
break;
case "SimpleRNN":
descriptor = new SimpleRNN(
(int)layer.SelectToken("units"),
(int)layer.SelectToken("input_dim"),
ANR((string)layer.SelectToken("activation")));
break;
case "LSTM":
descriptor = new LSTM(
(int)layer.SelectToken("units"),
(int)layer.SelectToken("input_dim"),
ANR((string)layer.SelectToken("activation")),
ANR((string)layer.SelectToken("rec_act")));
break;
case "GRU":
descriptor = new GRU(
(int)layer.SelectToken("units"),
(int)layer.SelectToken("input_dim"),
ANR((string)layer.SelectToken("activation")),
ANR((string)layer.SelectToken("rec_act")));
break;
case "ELu":
descriptor = new ELu(1);
break;
case "HardSigmoid":
descriptor = new HardSigmoid();
break;
case "ReLu":
descriptor = new ReLu();
break;
case "Sigmoid":
descriptor = new Sigmoid();
break;
case "Flatten":
descriptor = new Flatten();
break;
case "Softmax":
descriptor = new Softmax();
break;
case "SoftPlus":
descriptor = new SoftPlus();
break;
case "SoftSign":
descriptor = new Softsign();
break;
case "TanH":
descriptor = new TanH();
break;
case "LeakyReLu":
descriptor = new LeakyReLu(1);
break;
default:
throw new Exception("Unknown layer type!");
}
return(descriptor);
}).ToList();
return(dscps);
}
public static void Run(bool verbose)
{
Stopwatch sw = new Stopwatch();
NdArray inputArrayCpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
NdArray inputArrayGpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
Ensure.Argument(inputArrayGpu).NotNull();
Ensure.Argument(inputArrayCpu).NotNull();
//Linear
Linear linear = new Linear(verbose, INPUT_SIZE, OUTPUT_SIZE);
if (verbose)
{
RILogManager.Default?.EnterMethod(linear.Name);
}
sw.Restart();
NdArray[] gradArrayCpu = linear.Forward(verbose, inputArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
Ensure.Argument(gradArrayCpu).NotNull();
gradArrayCpu[0].Grad = gradArrayCpu[0].Data; // Use Data as Grad
sw.Restart();
linear.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (linear.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = linear.Forward(verbose, inputArrayGpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
linear.Backward(verbose, gradArrayGpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
}
if (verbose)
{
RILogManager.Default?.ExitMethod(linear.Name);
}
//Tanh
Tanh tanh = new Tanh();
if (verbose)
{
RILogManager.Default?.EnterMethod(tanh.Name);
}
sw.Restart();
gradArrayCpu = tanh.Forward(verbose, inputArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
tanh.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (tanh.SetGpuEnable(true))
{
HandleGPU(verbose, sw, tanh, inputArrayGpu);
}
if (verbose)
{
RILogManager.Default?.ExitMethod(tanh.Name);
}
//Sigmoid
Sigmoid sigmoid = new Sigmoid();
if (verbose)
{
RILogManager.Default?.EnterMethod(sigmoid.Name);
}
sw.Restart();
gradArrayCpu = sigmoid.Forward(verbose, inputArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sigmoid.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (sigmoid.SetGpuEnable(true))
{
HandleGPU(verbose, sw, sigmoid, inputArrayGpu);
}
if (verbose)
{
RILogManager.Default?.ExitMethod(tanh.Name);
}
//Softmax
Softmax sm = new Softmax();
RILogManager.Default?.EnterMethod(sm.Name);
sw.Restart();
gradArrayCpu = sm.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sm.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (verbose)
{
RILogManager.Default?.ExitMethod(sm.Name);
}
//Softplus
Softplus sp = new Softplus();
if (verbose)
{
RILogManager.Default?.EnterMethod(sp.Name);
}
sw.Restart();
gradArrayCpu = sp.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sp.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
RILogManager.Default?.ExitMethod(sp.Name);
//ReLU
ReLU relu = new ReLU();
RILogManager.Default?.EnterMethod(relu.Name);
sw.Restart();
gradArrayCpu = relu.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
relu.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (relu.SetGpuEnable(true))
{
HandleGPU(verbose, sw, relu, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(relu.Name);
//LeakyReLU
LeakyReLU leakyRelu = new LeakyReLU();
RILogManager.Default?.EnterMethod(leakyRelu.Name);
sw.Restart();
gradArrayCpu = leakyRelu.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
leakyRelu.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (leakyRelu.SetGpuEnable(true))
{
HandleGPU(verbose, sw, leakyRelu, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(leakyRelu.Name);
//ReLuTanh
ReLuTanh rth = new ReLuTanh();
RILogManager.Default?.EnterMethod(rth.Name);
sw.Restart();
gradArrayCpu = rth.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
rth.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (rth.SetGpuEnable(true))
{
HandleGPU(verbose, sw, rth, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(rth.Name);
////Swish
//Swish swi = new Swish();
//RILogManager.Default?.SendDebug(swi.Name);
//sw.Restart();
//gradArrayCpu = swi.Forward(inputArrayCpu);
//sw.Stop();
//RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
//gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
//sw.Restart();
//swi.Backward(gradArrayCpu);
//sw.Stop();
//RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
NdArray inputImageArrayGpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
NdArray inputImageArrayCpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
//MaxPooling
MaxPooling maxPooling = new MaxPooling(3);
RILogManager.Default?.EnterMethod(maxPooling.Name);
sw.Restart();
NdArray[] gradImageArrayCpu = maxPooling.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
maxPooling.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (maxPooling.SetGpuEnable(true))
{
sw.Restart();
maxPooling.Forward(verbose, inputImageArrayGpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
// There is no implementation for memory transfer only
RILogManager.Default?.SendDebug("Backward[Gpu] : None");
}
RILogManager.Default?.ExitMethod(maxPooling.Name);
//AvgPooling
AveragePooling avgPooling = new AveragePooling(3);
RILogManager.Default?.EnterMethod(avgPooling.Name);
sw.Restart();
gradImageArrayCpu = avgPooling.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
avgPooling.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
RILogManager.Default?.ExitMethod(avgPooling.Name);
//Conv2D
Convolution2D conv2d = new Convolution2D(verbose, 3, 3, 3);
RILogManager.Default?.EnterMethod(conv2d.Name);
sw.Restart();
gradImageArrayCpu = conv2d.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
conv2d.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (conv2d.SetGpuEnable(true))
{
HandleGPU(verbose, sw, conv2d, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(conv2d.Name);
//Deconv2D
Deconvolution2D deconv2d = new Deconvolution2D(verbose, 3, 3, 3);
RILogManager.Default?.EnterMethod(deconv2d.Name);
sw.Restart();
gradImageArrayCpu = deconv2d.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
deconv2d.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (deconv2d.SetGpuEnable(true))
{
HandleGPU(verbose, sw, deconv2d, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(deconv2d.Name);
//Dropout
Dropout dropout = new Dropout();
RILogManager.Default?.EnterMethod(dropout.Name);
sw.Restart();
gradArrayCpu = dropout.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
dropout.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (dropout.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = dropout.Forward(verbose, inputArrayGpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
dropout.Backward(verbose, gradArrayGpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
RILogManager.Default?.ExitMethod(dropout.Name);
//ArcSinH
ArcSinH a = new ArcSinH();
RILogManager.Default?.EnterMethod(a.Name);
sw.Restart();
gradArrayCpu = a.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
a.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (a.SetGpuEnable(true))
{
HandleGPU(verbose, sw, a, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(a.Name);
//ELU
ELU e = new ELU();
RILogManager.Default?.EnterMethod(e.Name);
sw.Restart();
gradArrayCpu = e.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
e.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
RILogManager.Default?.ExitMethod(e.Name);
//LeakyReluShifted
LeakyReLUShifted lrs = new LeakyReLUShifted();
RILogManager.Default?.EnterMethod(lrs.Name);
sw.Restart();
gradArrayCpu = lrs.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
lrs.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (lrs.SetGpuEnable(true))
{
HandleGPU(verbose, sw, lrs, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(lrs.Name);
//Logistic
LogisticFunction lf = new LogisticFunction();
RILogManager.Default?.EnterMethod(lf.Name);
sw.Restart();
gradArrayCpu = lf.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
lf.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (lf.SetGpuEnable(true))
{
HandleGPU(verbose, sw, lf, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(lf.Name);
//MaxMinusOne
MaxMinusOne mmo = new MaxMinusOne();
RILogManager.Default?.EnterMethod(mmo.Name);
sw.Restart();
gradArrayCpu = mmo.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
mmo.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (mmo.SetGpuEnable(true))
{
HandleGPU(verbose, sw, mmo, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(mmo.Name);
//ScaledELU
ScaledELU se = new ScaledELU();
RILogManager.Default?.EnterMethod(se.Name);
sw.Restart();
gradArrayCpu = se.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
se.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (se.SetGpuEnable(true))
{
HandleGPU(verbose, sw, se, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(se.Name);
//Sine
Sine s = new Sine();
RILogManager.Default?.EnterMethod(s.Name);
sw.Restart();
gradArrayCpu = s.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
s.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (s.SetGpuEnable(true))
{
HandleGPU(verbose, sw, s, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(s.Name);
}
