private static double[][] sum(params int[] axes) { var arr = new[] { /* total: * /* */1.0, 2.0, 3.0, /* 6.0 */ /* */ 4.0, 5.0, 6.0, /* 15.0 */ /* total: 5.0, 7.0, 9.0 21.0 */ }; var shape = NDShape.CreateNDShape(new[] { 2, 3 }); Value vx = Value.CreateBatch(shape, arr, DeviceDescriptor.CPUDevice, readOnly: true); Variable x = Variable.InputVariable(shape, CNTK.DataType.Double, name: "input"); CNTK.Function f; if (axes == null) { f = CNTKLib.Alias(x); } else { var axisVector = new AxisVector(axes.Select(ax => new Axis(ax)).ToArray()); f = CNTKLib.ReduceSum(x, axis: axisVector); } var inputs = new Dictionary <Variable, Value>() { { x, vx } }; var outputs = new Dictionary <Variable, Value>() { { f, null } }; f.Evaluate(inputs, outputs, DeviceDescriptor.CPUDevice); var r = outputs[f].GetDenseData <double>((Variable)f); return(r.Select(ri => ri.ToArray()).ToArray()); }
/// <summary> /// Adds a pulling layer for a two-dimensional vector. If the previous layer has a non-two-dimensional output, an exception is thrown /// </summary> /// <param name="poolingWindowWidth">Pulling window width</param> /// <param name="poolingWindowHeight">Pulling window height</param> /// <param name="hStride">Horizontal displacement step of the pulling window (according to matrix columns)</param> /// <param name="vStride">Step of shifting the pulling window vertically (along the rows of the matrix)</param> /// <param name="poolingType">Type of pulling. Maximum or medium</param> /// <param name="name"></param> public static Function Build(Variable input, int poolingWindowWidth, int poolingWindowHeight, int hStride, int vStride, PoolingType poolingType, string name) { var pooling = CNTKLib.Pooling(input, poolingType, new int[] { poolingWindowWidth, poolingWindowHeight }, new int[] { hStride, vStride }, new bool[] { true }); return(CNTKLib.Alias(pooling, name)); }
static void Main(string[] args) { const bool useSparse = false; const int vectorSize = 300; const int vocabularySize = 6000; const float xMax = 100f; const float alphaOrder = 0.75f; const int imagimableBatchSize = 1; var device = DeviceDescriptor.GPUDevice(0); var scalarDimension = new[] { imagimableBatchSize, 1 }; var matrixSize = new[] { vectorSize, vocabularySize }; var vectorDimension = new[] { vocabularySize, 1 }; var iterationScalarShape = NDShape.CreateNDShape(scalarDimension); var iterationMatrixShape = NDShape.CreateNDShape(matrixSize); var iterationVectorShape = NDShape.CreateNDShape(vectorDimension); var coOccurrences = Variable.InputVariable(iterationScalarShape, DataType.Float, "coOccurrences - " + vocabularySize, null, false); var columns = Variable.InputVariable(iterationScalarShape, DataType.Float, "columns - " + vocabularySize, null, false); var rows = Variable.InputVariable(iterationScalarShape, DataType.Float, "rows - " + vocabularySize, null, false); var mainVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device); PrintDim(mainVectors, nameof(mainVectors)); var contextVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device); PrintDim(contextVectors, nameof(contextVectors)); var mainBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device); PrintDim(mainBiases, nameof(mainBiases)); var contextBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device); PrintDim(contextBiases, nameof(contextBiases)); var one = new Constant(iterationScalarShape, DataType.Float, 1d, device); PrintDim(one, nameof(one)); var xmax = new Constant(iterationScalarShape, DataType.Float, xMax, device); PrintDim(xmax, nameof(xmax)); var alpha = new Constant(iterationScalarShape, DataType.Float, alphaOrder, device); PrintDim(alpha, nameof(alpha)); var divide = CNTKLib.ElementDivide(coOccurrences, xmax); PrintDim(divide, nameof(divide)); var pow = CNTKLib.Pow(divide, alpha); PrintDim(pow, nameof(pow)); var weight = CNTKLib.ElementMin(one, pow, "min"); PrintDim(weight, nameof(weight)); var oneHotRow = CNTKLib.OneHotOp(rows, vocabularySize, useSparse, new Axis(0)); PrintDim(oneHotRow, nameof(oneHotRow)); var oneHotColumn = CNTKLib.OneHotOp(columns, vocabularySize, useSparse, new Axis(0)); PrintDim(oneHotColumn, nameof(oneHotColumn)); var mainVector = CNTKLib.Alias(CNTKLib.Times(mainVectors, oneHotColumn)); PrintDim(mainVector, nameof(mainVector)); var contextVector = CNTKLib.Alias(CNTKLib.Times(contextVectors, oneHotRow)); PrintDim(contextVector, nameof(contextVector)); var mainBias = CNTKLib.Alias(CNTKLib.TransposeTimes(mainBiases, oneHotColumn)); PrintDim(mainBias, nameof(mainBias)); var contextBias = CNTKLib.Alias(CNTKLib.TransposeTimes(contextBiases, oneHotRow)); PrintDim(contextBias, nameof(contextBias)); var model = CNTKLib.ElementTimes(mainVector, contextVector); PrintDim(model, "CNTKLib.ElementTimes(mainVector, contextVector)"); model = CNTKLib.ReduceSum(model, new Axis(0)); PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(0))"); model = CNTKLib.Plus(model, mainBias); PrintDim(model, "CNTKLib.Plus(model, mainBias)"); model = CNTKLib.Plus(model, contextBias); PrintDim(model, "CNTKLib.Plus(model, contextBias)"); model = CNTKLib.Minus(model, CNTKLib.Log(coOccurrences)); PrintDim(model, "CNTKLib.Minus(model, CNTKLib.Log(coOccurrences))"); model = CNTKLib.Square(model); PrintDim(model, "CNTKLib.Square(model)"); model = CNTKLib.ElementTimes(model, weight); PrintDim(model, "CNTKLib.ElementTimes(model, weight)"); model = CNTKLib.ReduceSum(model, new Axis(1)); PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(1))"); var thisBatchShape = NDShape.CreateNDShape(new[] { imagimableBatchSize }); var parameterVector = new ParameterVector(model.Parameters().ToList()); var learner = CNTKLib.SGDLearner( parameterVector, new TrainingParameterScheduleDouble(0.1, (uint)(vocabularySize * vocabularySize))); var learners = new LearnerVector() { learner }; var trainer = CNTKLib.CreateTrainer(model, model, model, learners); var count = (int)(vocabularySize * vocabularySize * 0.2d * 0.2d); var floats = GetRandomFloats(count).ToArray(); var fColumns = GetRandomInts(count, 0, vocabularySize).ToArray(); var fRows = GetRandomInts(count, 0, vocabularySize).ToArray(); const int batchSize = 10000; var all = floats.Zip(fColumns, (f, c) => (f: f, c: c)).Zip(fRows, (tuple, r) => (tuple.c, tuple.c, r)) .ToObservable() .Buffer(batchSize) .Select(x => (f: x.Select(y => y.Item1).ToArray(), c: x.Select(y => y.Item2).ToArray(), r: x.Select(y => y.Item3).ToArray())) .ToArray() .Wait(); Console.WriteLine($"count: {count}"); var stopwatch = new Stopwatch(); stopwatch.Start(); for (var e = 0; e < 3; e++) { for (var i = 0; i < all.Length; i++) { var valueTuple = all[i]; var cooccurenceValue = Value.CreateBatch(thisBatchShape, valueTuple.f, device); var columnsValue = Value.CreateBatch(thisBatchShape, valueTuple.c, device); var rowsValue = Value.CreateBatch(thisBatchShape, valueTuple.r, device); var trainDictionary = new Dictionary <Variable, Value> { { coOccurrences, cooccurenceValue }, { columns, columnsValue }, { rows, rowsValue } }; trainer.TrainMinibatch(trainDictionary, false, device); if (i % 100 == 0) { Console.WriteLine($"e: {e}\ti: {stopwatch.Elapsed:g}"); } } } stopwatch.Stop(); Console.WriteLine($"success: {stopwatch.Elapsed:g}"); }