private static double[][] sum(params int[] axes)
{
var arr = new[]
{
/* total:
* /* */1.0, 2.0, 3.0, /* 6.0 */
/* */ 4.0, 5.0, 6.0, /* 15.0 */
/* total: 5.0, 7.0, 9.0 21.0 */
};
var shape = NDShape.CreateNDShape(new[] { 2, 3 });
Value vx = Value.CreateBatch(shape, arr, DeviceDescriptor.CPUDevice, readOnly: true);
Variable x = Variable.InputVariable(shape, CNTK.DataType.Double, name: "input");
CNTK.Function f;
if (axes == null)
{
f = CNTKLib.Alias(x);
}
else
{
var axisVector = new AxisVector(axes.Select(ax => new Axis(ax)).ToArray());
f = CNTKLib.ReduceSum(x, axis: axisVector);
}
var inputs = new Dictionary <Variable, Value>()
{
{ x, vx }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.CPUDevice);
var r = outputs[f].GetDenseData <double>((Variable)f);
return(r.Select(ri => ri.ToArray()).ToArray());
}
/// <summary>
/// Adds a pulling layer for a two-dimensional vector. If the previous layer has a non-two-dimensional output, an exception is thrown
/// </summary>
/// <param name="poolingWindowWidth">Pulling window width</param>
/// <param name="poolingWindowHeight">Pulling window height</param>
/// <param name="hStride">Horizontal displacement step of the pulling window (according to matrix columns)</param>
/// <param name="vStride">Step of shifting the pulling window vertically (along the rows of the matrix)</param>
/// <param name="poolingType">Type of pulling. Maximum or medium</param>
/// <param name="name"></param>
public static Function Build(Variable input, int poolingWindowWidth, int poolingWindowHeight, int hStride, int vStride, PoolingType poolingType, string name)
{
var pooling = CNTKLib.Pooling(input, poolingType, new int[] { poolingWindowWidth, poolingWindowHeight }, new int[] { hStride, vStride }, new bool[] { true });
return(CNTKLib.Alias(pooling, name));
}
static void Main(string[] args)
{
const bool useSparse = false;
const int vectorSize = 300;
const int vocabularySize = 6000;
const float xMax = 100f;
const float alphaOrder = 0.75f;
const int imagimableBatchSize = 1;
var device = DeviceDescriptor.GPUDevice(0);
var scalarDimension = new[] { imagimableBatchSize, 1 };
var matrixSize = new[] { vectorSize, vocabularySize };
var vectorDimension = new[] { vocabularySize, 1 };
var iterationScalarShape = NDShape.CreateNDShape(scalarDimension);
var iterationMatrixShape = NDShape.CreateNDShape(matrixSize);
var iterationVectorShape = NDShape.CreateNDShape(vectorDimension);
var coOccurrences = Variable.InputVariable(iterationScalarShape, DataType.Float, "coOccurrences - " + vocabularySize, null, false);
var columns = Variable.InputVariable(iterationScalarShape, DataType.Float, "columns - " + vocabularySize, null, false);
var rows = Variable.InputVariable(iterationScalarShape, DataType.Float, "rows - " + vocabularySize, null, false);
var mainVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device);
PrintDim(mainVectors, nameof(mainVectors));
var contextVectors = new Parameter(iterationMatrixShape, DataType.Float, 0d, device);
PrintDim(contextVectors, nameof(contextVectors));
var mainBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device);
PrintDim(mainBiases, nameof(mainBiases));
var contextBiases = new Parameter(iterationVectorShape, DataType.Float, 0d, device);
PrintDim(contextBiases, nameof(contextBiases));
var one = new Constant(iterationScalarShape, DataType.Float, 1d, device);
PrintDim(one, nameof(one));
var xmax = new Constant(iterationScalarShape, DataType.Float, xMax, device);
PrintDim(xmax, nameof(xmax));
var alpha = new Constant(iterationScalarShape, DataType.Float, alphaOrder, device);
PrintDim(alpha, nameof(alpha));
var divide = CNTKLib.ElementDivide(coOccurrences, xmax);
PrintDim(divide, nameof(divide));
var pow = CNTKLib.Pow(divide, alpha);
PrintDim(pow, nameof(pow));
var weight = CNTKLib.ElementMin(one, pow, "min");
PrintDim(weight, nameof(weight));
var oneHotRow = CNTKLib.OneHotOp(rows, vocabularySize, useSparse, new Axis(0));
PrintDim(oneHotRow, nameof(oneHotRow));
var oneHotColumn = CNTKLib.OneHotOp(columns, vocabularySize, useSparse, new Axis(0));
PrintDim(oneHotColumn, nameof(oneHotColumn));
var mainVector = CNTKLib.Alias(CNTKLib.Times(mainVectors, oneHotColumn));
PrintDim(mainVector, nameof(mainVector));
var contextVector = CNTKLib.Alias(CNTKLib.Times(contextVectors, oneHotRow));
PrintDim(contextVector, nameof(contextVector));
var mainBias = CNTKLib.Alias(CNTKLib.TransposeTimes(mainBiases, oneHotColumn));
PrintDim(mainBias, nameof(mainBias));
var contextBias = CNTKLib.Alias(CNTKLib.TransposeTimes(contextBiases, oneHotRow));
PrintDim(contextBias, nameof(contextBias));
var model = CNTKLib.ElementTimes(mainVector, contextVector);
PrintDim(model, "CNTKLib.ElementTimes(mainVector, contextVector)");
model = CNTKLib.ReduceSum(model, new Axis(0));
PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(0))");
model = CNTKLib.Plus(model, mainBias);
PrintDim(model, "CNTKLib.Plus(model, mainBias)");
model = CNTKLib.Plus(model, contextBias);
PrintDim(model, "CNTKLib.Plus(model, contextBias)");
model = CNTKLib.Minus(model, CNTKLib.Log(coOccurrences));
PrintDim(model, "CNTKLib.Minus(model, CNTKLib.Log(coOccurrences))");
model = CNTKLib.Square(model);
PrintDim(model, "CNTKLib.Square(model)");
model = CNTKLib.ElementTimes(model, weight);
PrintDim(model, "CNTKLib.ElementTimes(model, weight)");
model = CNTKLib.ReduceSum(model, new Axis(1));
PrintDim(model, "CNTKLib.ReduceSum(model, new Axis(1))");
var thisBatchShape = NDShape.CreateNDShape(new[] { imagimableBatchSize });
var parameterVector = new ParameterVector(model.Parameters().ToList());
var learner = CNTKLib.SGDLearner(
parameterVector,
new TrainingParameterScheduleDouble(0.1, (uint)(vocabularySize * vocabularySize)));
var learners = new LearnerVector()
{
learner
};
var trainer = CNTKLib.CreateTrainer(model, model, model, learners);
var count = (int)(vocabularySize * vocabularySize * 0.2d * 0.2d);
var floats = GetRandomFloats(count).ToArray();
var fColumns = GetRandomInts(count, 0, vocabularySize).ToArray();
var fRows = GetRandomInts(count, 0, vocabularySize).ToArray();
const int batchSize = 10000;
var all = floats.Zip(fColumns, (f, c) => (f: f, c: c)).Zip(fRows, (tuple, r) => (tuple.c, tuple.c, r))
.ToObservable()
.Buffer(batchSize)
.Select(x => (f: x.Select(y => y.Item1).ToArray(), c: x.Select(y => y.Item2).ToArray(), r: x.Select(y => y.Item3).ToArray()))
.ToArray()
.Wait();
Console.WriteLine($"count: {count}");
var stopwatch = new Stopwatch();
stopwatch.Start();
for (var e = 0; e < 3; e++)
{
for (var i = 0; i < all.Length; i++)
{
var valueTuple = all[i];
var cooccurenceValue = Value.CreateBatch(thisBatchShape, valueTuple.f, device);
var columnsValue = Value.CreateBatch(thisBatchShape, valueTuple.c, device);
var rowsValue = Value.CreateBatch(thisBatchShape, valueTuple.r, device);
var trainDictionary = new Dictionary <Variable, Value>
{
{ coOccurrences, cooccurenceValue },
{ columns, columnsValue },
{ rows, rowsValue }
};
trainer.TrainMinibatch(trainDictionary, false, device);
if (i % 100 == 0)
{
Console.WriteLine($"e: {e}\ti: {stopwatch.Elapsed:g}");
}
}
}
stopwatch.Stop();
Console.WriteLine($"success: {stopwatch.Elapsed:g}");
}