private static double[][] sum(params int[] axes)
        {
            var arr = new[]
            {
                /*                            total:
                 * /*       */1.0, 2.0, 3.0, /*  6.0 */
                /*       */ 4.0, 5.0, 6.0, /* 15.0 */
                /* total:   5.0, 7.0, 9.0     21.0 */
            };

            var      shape = NDShape.CreateNDShape(new[] { 2, 3 });
            Value    vx    = Value.CreateBatch(shape, arr, DeviceDescriptor.CPUDevice, readOnly: true);
            Variable x     = Variable.InputVariable(shape, CNTK.DataType.Double, name: "input");

            CNTK.Function f;
            if (axes == null)
            {
                f = CNTKLib.Alias(x);
            }
            else
            {
                var axisVector = new AxisVector(axes.Select(ax => new Axis(ax)).ToArray());
                f = CNTKLib.ReduceSum(x, axis: axisVector);
            }

            var inputs = new Dictionary <Variable, Value>()
            {
                { x, vx }
            };
            var outputs = new Dictionary <Variable, Value>()
            {
                { f, null }
            };

            f.Evaluate(inputs, outputs, DeviceDescriptor.CPUDevice);
            var r = outputs[f].GetDenseData <double>((Variable)f);

            return(r.Select(ri => ri.ToArray()).ToArray());
        }
Beispiel #2
0
        /// <summary>
        /// Adds a pulling layer for a two-dimensional vector. If the previous layer has a non-two-dimensional output, an exception is thrown
        /// </summary>
        /// <param name="poolingWindowWidth">Pulling window width</param>
        /// <param name="poolingWindowHeight">Pulling window height</param>
        /// <param name="hStride">Horizontal displacement step of the pulling window (according to matrix columns)</param>
        /// <param name="vStride">Step of shifting the pulling window vertically (along the rows of the matrix)</param>
        /// <param name="poolingType">Type of pulling. Maximum or medium</param>
        /// <param name="name"></param>
        public static Function Build(Variable input, int poolingWindowWidth, int poolingWindowHeight, int hStride, int vStride, PoolingType poolingType, string name)
        {
            var pooling = CNTKLib.Pooling(input, poolingType, new int[] { poolingWindowWidth, poolingWindowHeight }, new int[] { hStride, vStride }, new bool[] { true });

            return(CNTKLib.Alias(pooling, name));
        }
        static void Main(string[] args)
        {
            const bool useSparse = false;

            const int   vectorSize          = 300;
            const int   vocabularySize      = 6000;
            const float xMax                = 100f;
            const float alphaOrder          = 0.75f;
            const int   imagimableBatchSize = 1;

            var device = DeviceDescriptor.GPUDevice(0);

            var scalarDimension = new[] { imagimableBatchSize, 1 };
            var matrixSize      = new[] { vectorSize, vocabularySize };
            var vectorDimension = new[] { vocabularySize, 1 };

            var iterationScalarShape = NDShape.CreateNDShape(scalarDimension);
            var iterationMatrixShape = NDShape.CreateNDShape(matrixSize);
            var iterationVectorShape = NDShape.CreateNDShape(vectorDimension);

            var coOccurrences = Variable.InputVariable(iterationScalarShape, DataType.Float, "coOccurrences - " + vocabularySize, null, false);
            var columns       = Variable.InputVariable(iterationScalarShape, DataType.Float, "columns - " + vocabularySize, null, false);
            var rows          = Variable.InputVariable(iterationScalarShape, DataType.Float, "rows - " + vocabularySize, null, false);

            var mainVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device);

            PrintDim(mainVectors, nameof(mainVectors));
            var contextVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device);

            PrintDim(contextVectors, nameof(contextVectors));
            var mainBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device);

            PrintDim(mainBiases, nameof(mainBiases));
            var contextBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device);

            PrintDim(contextBiases, nameof(contextBiases));
            var one = new Constant(iterationScalarShape, DataType.Float, 1d, device);

            PrintDim(one, nameof(one));
            var xmax = new Constant(iterationScalarShape, DataType.Float, xMax, device);

            PrintDim(xmax, nameof(xmax));
            var alpha = new Constant(iterationScalarShape, DataType.Float, alphaOrder, device);

            PrintDim(alpha, nameof(alpha));


            var divide = CNTKLib.ElementDivide(coOccurrences, xmax);

            PrintDim(divide, nameof(divide));
            var pow = CNTKLib.Pow(divide, alpha);

            PrintDim(pow, nameof(pow));
            var weight = CNTKLib.ElementMin(one, pow, "min");

            PrintDim(weight, nameof(weight));

            var oneHotRow = CNTKLib.OneHotOp(rows, vocabularySize, useSparse, new Axis(0));

            PrintDim(oneHotRow, nameof(oneHotRow));
            var oneHotColumn = CNTKLib.OneHotOp(columns, vocabularySize, useSparse, new Axis(0));

            PrintDim(oneHotColumn, nameof(oneHotColumn));

            var mainVector = CNTKLib.Alias(CNTKLib.Times(mainVectors, oneHotColumn));

            PrintDim(mainVector, nameof(mainVector));
            var contextVector = CNTKLib.Alias(CNTKLib.Times(contextVectors, oneHotRow));

            PrintDim(contextVector, nameof(contextVector));

            var mainBias = CNTKLib.Alias(CNTKLib.TransposeTimes(mainBiases, oneHotColumn));

            PrintDim(mainBias, nameof(mainBias));

            var contextBias = CNTKLib.Alias(CNTKLib.TransposeTimes(contextBiases, oneHotRow));

            PrintDim(contextBias, nameof(contextBias));

            var model = CNTKLib.ElementTimes(mainVector, contextVector);

            PrintDim(model, "CNTKLib.ElementTimes(mainVector, contextVector)");

            model = CNTKLib.ReduceSum(model, new Axis(0));
            PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(0))");

            model = CNTKLib.Plus(model, mainBias);
            PrintDim(model, "CNTKLib.Plus(model, mainBias)");

            model = CNTKLib.Plus(model, contextBias);
            PrintDim(model, "CNTKLib.Plus(model, contextBias)");

            model = CNTKLib.Minus(model, CNTKLib.Log(coOccurrences));
            PrintDim(model, "CNTKLib.Minus(model, CNTKLib.Log(coOccurrences))");

            model = CNTKLib.Square(model);
            PrintDim(model, "CNTKLib.Square(model)");

            model = CNTKLib.ElementTimes(model, weight);
            PrintDim(model, "CNTKLib.ElementTimes(model, weight)");

            model = CNTKLib.ReduceSum(model, new Axis(1));
            PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(1))");

            var thisBatchShape = NDShape.CreateNDShape(new[] { imagimableBatchSize });

            var parameterVector = new ParameterVector(model.Parameters().ToList());

            var learner = CNTKLib.SGDLearner(
                parameterVector,
                new TrainingParameterScheduleDouble(0.1, (uint)(vocabularySize * vocabularySize)));

            var learners = new LearnerVector()
            {
                learner
            };
            var trainer = CNTKLib.CreateTrainer(model, model, model, learners);


            var count    = (int)(vocabularySize * vocabularySize * 0.2d * 0.2d);
            var floats   = GetRandomFloats(count).ToArray();
            var fColumns = GetRandomInts(count, 0, vocabularySize).ToArray();
            var fRows    = GetRandomInts(count, 0, vocabularySize).ToArray();

            const int batchSize = 10000;
            var       all       = floats.Zip(fColumns, (f, c) => (f: f, c: c)).Zip(fRows, (tuple, r) => (tuple.c, tuple.c, r))
                                  .ToObservable()
                                  .Buffer(batchSize)
                                  .Select(x => (f: x.Select(y => y.Item1).ToArray(), c: x.Select(y => y.Item2).ToArray(), r: x.Select(y => y.Item3).ToArray()))
                                  .ToArray()
                                  .Wait();

            Console.WriteLine($"count: {count}");

            var stopwatch = new Stopwatch();

            stopwatch.Start();
            for (var e = 0; e < 3; e++)
            {
                for (var i = 0; i < all.Length; i++)
                {
                    var valueTuple = all[i];

                    var cooccurenceValue = Value.CreateBatch(thisBatchShape, valueTuple.f, device);
                    var columnsValue     = Value.CreateBatch(thisBatchShape, valueTuple.c, device);
                    var rowsValue        = Value.CreateBatch(thisBatchShape, valueTuple.r, device);

                    var trainDictionary = new Dictionary <Variable, Value>
                    {
                        { coOccurrences, cooccurenceValue },
                        { columns, columnsValue },
                        { rows, rowsValue }
                    };


                    trainer.TrainMinibatch(trainDictionary, false, device);

                    if (i % 100 == 0)
                    {
                        Console.WriteLine($"e: {e}\ti: {stopwatch.Elapsed:g}");
                    }
                }
            }

            stopwatch.Stop();


            Console.WriteLine($"success: {stopwatch.Elapsed:g}");
        }