void create_network() { Console.WriteLine("Compute Device: " + computeDevice.AsString()); imageVariable = Util.inputVariable(new int[] { 28, 28, 1 }, "image_tensor"); categoricalVariable = Util.inputVariable(new int[] { 10 }, "label_tensor"); network = imageVariable; network = Layers.Convolution2D(network, 32, new int[] { 3, 3 }, computeDevice, CC.ReLU); network = CC.Pooling(network, C.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 }); network = Layers.Convolution2D(network, 64, new int[] { 3, 3 }, computeDevice, CC.ReLU); network = CC.Pooling(network, C.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 }); network = Layers.Convolution2D(network, 64, new int[] { 3, 3 }, computeDevice, CC.ReLU); network = Layers.Dense(network, 64, computeDevice, activation: CC.ReLU); network = Layers.Dense(network, 10, computeDevice); Logging.detailed_summary(network); Logging.log_number_of_parameters(network); loss_function = CC.CrossEntropyWithSoftmax(network, categoricalVariable); eval_function = CC.ClassificationError(network, categoricalVariable); learner = CC.AdamLearner( new C.ParameterVector(network.Parameters().ToArray()), new C.TrainingParameterScheduleDouble(0.001 * batch_size, (uint)batch_size), new C.TrainingParameterScheduleDouble(0.9), true, new C.TrainingParameterScheduleDouble(0.99)); trainer = CC.CreateTrainer(network, loss_function, eval_function, new C.LearnerVector(new C.Learner[] { learner })); evaluator = CC.CreateEvaluator(eval_function); }
public QNetworkConvSimple(int inputWidth, int inputHeight, int inputDepth, int actionSize, int[] filterSizes, int[] filterDepths, int[] strides, bool[] pooling, int densehiddenLayers, int densehiddenSize, bool denseUseBias, DeviceDescriptor device, float denseInitialWeightScale = 0.01f) { Device = device; StateSize = inputWidth * inputHeight * inputDepth; ActionSize = actionSize; InputDimension = new int[3] { inputWidth, inputHeight, inputDepth }; //create actor network part InputState = CNTKLib.InputVariable(InputDimension, DataType.Float); Debug.Assert(filterSizes.Length == strides.Length && filterDepths.Length == filterSizes.Length, "Length of filterSizes,strides and filterDepth are not the same"); var lastLayer = InputState; for (int i = 0; i < filterSizes.Length; ++i) { //conv layers. Use selu activaion and selu initlaization lastLayer = Layers.Convolution2D(lastLayer, filterDepths[i], filterSizes[i], filterSizes[i], device, strides[i], true, true, "QConv_" + i, Mathf.Sqrt((1.0f / (filterSizes[i] * filterSizes[i])))); lastLayer = new SELUDef().BuildNew(lastLayer, device, ""); //pooling if (pooling[i]) { lastLayer = CNTKLib.Pooling(lastLayer, PoolingType.Max, new int[] { 2, 2 }, new int[] { 2, 2 }, BoolVector.Repeat(true, 2), false, true, "pool2"); } } lastLayer = CNTKLib.Flatten(lastLayer, new Axis(3), "Flatten"); //dense layers var inputA = new InputLayerCNTKVar(lastLayer); var outputA = new OutputLayerDense(actionSize, null, OutputLayerDense.LossFunction.None); outputA.HasBias = false; outputA.InitialWeightScale = denseInitialWeightScale; SequentialNetworkDense qNetwork = new SequentialNetworkDense(inputA, LayerDefineHelper.DenseLayers(densehiddenLayers, densehiddenSize, denseUseBias, NormalizationMethod.None, 0, denseInitialWeightScale, new ReluDef()), outputA, device); //OutputQs = outputA.GetOutputVariable(); OutputQs = outputA.GetOutputVariable(); }
C.Function create_primary_cap(C.Function inputs, int dim_capsule, int n_channels, int[] kernel_size, int[] strides, bool pad) { var output = Layers.Convolution2D( inputs, dim_capsule * n_channels, kernel_size, computeDevice, strides: strides, use_padding: pad, name: "primarycap_conv2d"); var outputShape = output.Output.Shape.Dimensions; System.Diagnostics.Debug.Assert((outputShape[2] == 256) && (outputShape[1] == 6) && (outputShape[0] == 6)); var num_rows = (int)(Util.np_prod(outputShape.ToArray()) / dim_capsule); var target_shape = new int[] { num_rows, dim_capsule }; var outputs = CC.Reshape(output, target_shape, name: "primarycap_reshape"); var rtrn = squash(outputs, name: "primarycap_squash", axis: 1); return(rtrn); }
/// <summary> /// Create a Universal style transfer decoder. There are 5 decoders in UST, use index to specify which one to create. /// </summary> /// <param name="imageDimension">image dimension for the decoder to work with. Note that it might be different from the decoder input dimemensions</param> /// <param name="index"></param> /// <returns></returns> protected Function CreateDecoders(Vector2Int imageDimension, PassIndex index) { Function encoderLayers; int ind = (int)index + 1; //1 based index for the names Vector2Int inputDims = imageDimension; int[] xOffsets = new int[5]; int[] yOffsets = new int[5]; for (int i = 1; i < ind; ++i) { var temp = new Vector2Int(Mathf.CeilToInt(inputDims.x / 2.0f), Mathf.CeilToInt(inputDims.y / 2.0f)); if (Mathf.Abs(inputDims.x / 2.0f - temp.x) > 0.00001f) { xOffsets[i] = 1; } if (Mathf.Abs(inputDims.y / 2.0f - temp.y) > 0.00001f) { yOffsets[i] = 1; } inputDims = temp; } //set the channel number int inputChannels = 64; switch (index) { case PassIndex.PassOne: inputChannels = 64; break; case PassIndex.PassTwo: inputChannels = 128; break; case PassIndex.PassThree: inputChannels = 256; break; case PassIndex.PassFour: inputChannels = 512; break; case PassIndex.PassFive: inputChannels = 512; break; default: inputChannels = 64; break; } //input variables Variable prev = Variable.InputVariable(new int[] { inputDims.x, inputDims.y, inputChannels }, DataType.Float, "input"); //encoderLayers["input"] = prev; if (ind >= 5) { //decoder 5 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "de" + ind + "conv5_1"); prev = CNTKLib.ReLU(prev); prev = Layers.Upsample2D2(prev, xOffsets[4], yOffsets[4]); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "de" + ind + "conv4_4"); prev = CNTKLib.ReLU(prev); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "de" + ind + "conv4_3"); prev = CNTKLib.ReLU(prev); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "de" + ind + "conv4_2"); prev = CNTKLib.ReLU(prev); } if (ind >= 4) { //decoder 4 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "de" + ind + "conv4_1"); prev = CNTKLib.ReLU(prev); prev = Layers.Upsample2D2(prev, xOffsets[3], yOffsets[3]); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "de" + ind + "conv3_4"); prev = CNTKLib.ReLU(prev); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "de" + ind + "conv3_3"); prev = CNTKLib.ReLU(prev); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "de" + ind + "conv3_2"); prev = CNTKLib.ReLU(prev); } if (ind >= 3) { //decoder 3 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 128, 3, 3, device, 1, false, true, "de" + ind + "conv3_1"); prev = CNTKLib.ReLU(prev); prev = Layers.Upsample2D2(prev, xOffsets[2], yOffsets[2]); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 128, 3, 3, device, 1, false, true, "de" + ind + "conv2_2"); prev = CNTKLib.ReLU(prev); } if (ind >= 2) { //decoder 2 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 64, 3, 3, device, 1, false, true, "de" + ind + "conv2_1"); prev = CNTKLib.ReLU(prev); prev = Layers.Upsample2D2(prev, xOffsets[1], yOffsets[1]); prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 64, 3, 3, device, 1, false, true, "de" + ind + "conv1_2"); prev = CNTKLib.ReLU(prev); } //decoder 1 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 3, 3, 3, device, 1, false, true, "de" + ind + "conv1_1"); prev = CNTKLib.ReLU(prev, "output"); //encoderLayers["output"] = prev; encoderLayers = prev; return(encoderLayers); }
/// <summary> /// Create the vgg19 convolutional encoders for UST model /// </summary> /// <param name="imageDimension"></param> /// <returns></returns> protected Function CreateEncoders(Vector2Int imageDimension) { Function encoderLayers; VariableVector outputs = new VariableVector(); //input variables Variable prev = Variable.InputVariable(new int[] { imageDimension.x, imageDimension.y, 3 }, DataType.Float, "input"); //encoderLayers["input"] = prev; //vgg preprocessing prev = Layers.Convolution2D(prev, 3, 1, 1, device, 1, false, true, "conv0_preprocessing"); //encoderLayers["conv0_preprocessing"] = prev; //----conv1---- //conv1_1 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 64, 3, 3, device, 1, false, true, "conv1_1"); prev = CNTKLib.ReLU(prev, "relu1_1"); //encoderLayers["relu1_1"] = prev; outputs.Add(prev); //conv1_2 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 64, 3, 3, device, 1, false, true, "conv1_2"); prev = CNTKLib.ReLU(prev, "relu1_2"); //maxpooling 1 prev = CNTKLib.Pooling(prev, PoolingType.Max, new int[] { 2, 2 }, new int[] { 2, 2 }, BoolVector.Repeat(true, 2), false, true, "pool1"); //----conv2---- //conv2_1 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 128, 3, 3, device, 1, false, true, "conv2_1"); prev = CNTKLib.ReLU(prev, "relu2_1"); outputs.Add(prev); // encoderLayers["relu2_1"] = prev; //conv2_2 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 128, 3, 3, device, 1, false, true, "conv2_2"); prev = CNTKLib.ReLU(prev, "relu2_2"); //maxpooling 2 prev = CNTKLib.Pooling(prev, PoolingType.Max, new int[] { 2, 2 }, new int[] { 2, 2 }, BoolVector.Repeat(true, 2), false, true, "pool2"); //----conv3---- //conv3_1 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "conv3_1"); prev = CNTKLib.ReLU(prev, "relu3_1"); outputs.Add(prev); //encoderLayers["relu3_1"] = prev; //conv3_2 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "conv3_2"); prev = CNTKLib.ReLU(prev, "relu3_2"); //conv3_3 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "conv3_3"); prev = CNTKLib.ReLU(prev, "relu3_3"); //conv3_4 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 256, 3, 3, device, 1, false, true, "conv3_4"); prev = CNTKLib.ReLU(prev, "relu3_4"); //maxpooling 3 prev = CNTKLib.Pooling(prev, PoolingType.Max, new int[] { 2, 2 }, new int[] { 2, 2 }, BoolVector.Repeat(true, 2), false, true, "pool3"); //----conv4---- //conv4_1 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "conv4_1"); prev = CNTKLib.ReLU(prev, "relu4_1"); outputs.Add(prev); //encoderLayers["relu4_1"] = prev; //conv4_2 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "conv4_2"); prev = CNTKLib.ReLU(prev, "relu4_2"); //conv4_3 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "conv4_3"); prev = CNTKLib.ReLU(prev, "relu4_3"); //conv4_4 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "conv4_4"); prev = CNTKLib.ReLU(prev, "relu4_4"); //maxpooling 4 prev = CNTKLib.Pooling(prev, PoolingType.Max, new int[] { 2, 2 }, new int[] { 2, 2 }, BoolVector.Repeat(true, 2), false, true, "pool4"); //----conv5---- //conv5_1 prev = CNTKLib.Pad(prev, PaddingMode.REFLECTPAD, new SizeTVector(new uint[] { 1, 1, 0 }), new SizeTVector(new uint[] { 1, 1, 0 })); prev = Layers.Convolution2D(prev, 512, 3, 3, device, 1, false, true, "conv5_1"); prev = CNTKLib.ReLU(prev, "relu5_1"); outputs.Add(prev); encoderLayers = CNTKLib.Combine(outputs); //encoderLayers["relu5_1"] = prev; return(encoderLayers); }
void create_network() { imageVariable = Util.inputVariable(input_shape, "image"); var conv1 = Layers.Convolution2D( imageVariable, 256, new int[] { 9, 9 }, computeDevice, use_padding: false, activation: CC.ReLU, name: "conv1"); var primarycaps = create_primary_cap( conv1, dim_capsule: 8, n_channels: 32, kernel_size: new int[] { 9, 9 }, strides: new int[] { 2, 2 }, pad: false); var digitcaps = create_capsule_layer( primarycaps, num_capsule: 10, dim_capsule: 16, routings: routings, name: "digitcaps"); var out_caps = get_length_and_remove_last_dimension(digitcaps, name: "capsnet"); categoricalLabel = Util.inputVariable(new int[] { 10 }, "label"); var masked_by_y = get_mask_and_infer_from_last_dimension(digitcaps, CC.Combine(new C.VariableVector() { categoricalLabel })); var masked = get_mask_and_infer_from_last_dimension(digitcaps, null); var decoder = create_decoder(masked.Output.Shape.Dimensions.ToArray()); var decoder_output_training = Model.invoke_model(decoder, new C.Variable[] { masked_by_y }); var decoder_output_evaluation = Model.invoke_model(decoder, new C.Variable[] { masked }); network = CC.Combine(new C.VariableVector() { out_caps, decoder_output_training }, "overall_training_network"); Logging.log_number_of_parameters(network); // first component of the loss var y_true = categoricalLabel; var y_pred = out_caps; var digit_loss = CC.Plus( CC.ElementTimes(y_true, CC.Square(CC.ElementMax(DC(0), CC.Minus(DC(0.9), y_pred), ""))), CC.ElementTimes(DC(0.5), CC.ElementTimes(CC.Minus(DC(1), y_true), CC.Square(CC.ElementMax(DC(0), CC.Minus(y_pred, DC(0.1)), ""))))); digit_loss = CC.ReduceSum(digit_loss, C.Axis.AllStaticAxes()); // second component of the loss var num_pixels_at_output = Util.np_prod(decoder_output_training.Output.Shape.Dimensions.ToArray()); var squared_error = CC.SquaredError(decoder_output_training, imageVariable); var image_mse = CC.ElementDivide(squared_error, DC(num_pixels_at_output)); loss_function = CC.Plus(digit_loss, CC.ElementTimes(DC(0.35), image_mse)); eval_function = CC.ClassificationError(y_pred, y_true); learner = CC.AdamLearner( new C.ParameterVector(network.Parameters().ToArray()), new C.TrainingParameterScheduleDouble(0.001 * batch_size, (uint)batch_size), new C.TrainingParameterScheduleDouble(0.9), true, new C.TrainingParameterScheduleDouble(0.99)); trainer = CC.CreateTrainer(network, loss_function, eval_function, new C.LearnerVector(new C.Learner[] { learner })); evaluator = CC.CreateEvaluator(eval_function); }