public void TestWithParameter()
{
var input = Variable.InputVariable(new int[] { 1 }, DataType.Float, "input", new Axis[] { Axis.DefaultBatchAxis() });
var exp = input + Constant.Scalar(DataType.Float, 1);
var sampler = new ExpressionSampler("value", exp, input, 3);
var batch = sampler.GetNextMinibatch();
var data = batch["value"];
CollectionAssert.AreEqual(data.Shape.Dimensions.ToArray(), new int[] { 1, 3 });
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(ds.TypedData, new float[] { 1, 1, 1 });
batch = sampler.GetNextMinibatch();
data = batch["value"];
CollectionAssert.AreEqual(data.Shape.Dimensions.ToArray(), new int[] { 1, 3 });
ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(ds.TypedData, new float[] { 2, 2, 2 });
batch = sampler.GetNextMinibatch();
data = batch["value"];
CollectionAssert.AreEqual(data.Shape.Dimensions.ToArray(), new int[] { 1, 3 });
ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(ds.TypedData, new float[] { 3, 3, 3 });
}
public void TestMsgPackSamplerMultipleFiles()
{
var files = new string[2];
for (var i = 0; i < 2; ++i)
{
files[i] = Path.GetTempFileName();
var a = DataSourceFactory.Create(new float[] { i, i + 1, i + 2 }, new int[] { 3, 1 });
var dss = new DataSourceSet();
dss.Add("a", a);
using (var stream = new FileStream(files[i], FileMode.Create, FileAccess.Write))
{
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(files, 3, false, -1, 10, false, 10))
{
Assert.AreEqual(2, sampler.SampleCountPerEpoch);
var batch = sampler.GetNextMinibatch();
CollectionAssert.AreEqual(new int[] { 3, 3 }, batch.Features["a"].Shape.Dimensions.ToArray());
var values = DataSourceFactory.FromValue(batch.Features["a"]);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 1, 2, 3, 0, 1, 2 }, values.TypedData);
}
}
public void TestConvolutionTranspose2()
{
var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 2, 3 });
// var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4 }, new int[] { 2, 2, 1 });
var input = CNTKLib.InputVariable(new int[] { 2, 2, 3 }, DataType.Float);
var convolutionMap = new Parameter(new int[] { 2, 2, 6, 3 }, DataType.Float, CNTKLib.ConstantInitializer(1));
var f = CNTKLib.ConvolutionTranspose(convolutionMap, input, new int[] { 2, 2, 3 }, new BoolVector(new bool[] { true }), new BoolVector(new bool[] { true }), new int[] { 0 }, new int[] { 1 }, 1, 0, "");
var inputs = new Dictionary <Variable, Value>()
{
{ input, inputData.ToValue() }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f.Output, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.UseDefaultDevice());
var result = DataSourceFactory.FromValue(outputs[f.Output]);
CollectionAssert.AreEqual(new int[] { 3, 3, 6, 1, 1 }, result.Shape.Dimensions);
Assert.AreEqual(15, result.Data[0]);
Assert.AreEqual(15, result.Data[1]);
Assert.AreEqual(18, result.Data[2]);
Assert.AreEqual(15, result.Data[3]);
}
public void TestConvTrans()
{
var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4 }, new int[] { 2, 2, 1 });
var input = CNTKLib.InputVariable(new int[] { 2, 2, 1 }, DataType.Float);
var f = Composite.ConvolutionTranspose(input, new int[] { 2, 2 }, 1, null, CNTKLib.ConstantInitializer(1), new bool[] { true }, new int[] { 2, 2, 1 }, false, null, new int[] { 0 }, new int[] { 1 }, 1, 0, "");
var inputs = new Dictionary <Variable, Value>()
{
{ input, inputData.ToValue() }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f.Output, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.UseDefaultDevice());
var result = DataSourceFactory.FromValue(outputs[f.Output]);
CollectionAssert.AreEqual(new int[] { 3, 3, 1, 1, 1 }, result.Shape.Dimensions);
Assert.AreEqual(1, result.Data[0]);
Assert.AreEqual(1, result.Data[1]);
Assert.AreEqual(2, result.Data[2]);
Assert.AreEqual(1, result.Data[3]);
}
public void TestMinibatch()
{
var features = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, new int[] { 2, 1, -1 });
var dss = new Dictionary <string, IDataSource <float> >()
{
{ "input", features }
};
var sampler = new DataSourceSampler(dss, 2, false, true);
{
var batch = sampler.GetNextMinibatch();
// GC.Collect();
// GC.Collect();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(false, batch.SweepEnd);
var data = batch.Features["input"];
// var c1 = SharedPtrMethods.GetUseCountOf(data);
// var c2 = SharedPtrMethods.GetUseCountOf(data.data);
// var c3 = SharedPtrMethods.GetUseCountOf(data.data.Data);
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 3 }, ds.TypedData);
CollectionAssert.AreEqual(new int[] { 2, 1, 2 }, data.Shape.Dimensions.ToArray());
}
{
var batch = sampler.GetNextMinibatch();
// GC.Collect();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(true, batch.SweepEnd);
var data = batch.Features["input"];
// var c1 = SharedPtrMethods.GetUseCountOf(data);
// var c2 = SharedPtrMethods.GetUseCountOf(data.data);
// var c3 = SharedPtrMethods.GetUseCountOf(data.data.Data);
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(new float[] { 4, 5, 6, 7 }, ds.TypedData);
CollectionAssert.AreEqual(new int[] { 2, 1, 2 }, data.Shape.Dimensions.ToArray());
}
// When not randomized, remnant data that is smaller than the minibatch size is ignored.
{
var batch = sampler.GetNextMinibatch();
// GC.Collect();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(false, batch.SweepEnd);
var data = batch.Features["input"];
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 3 }, ds.TypedData);
CollectionAssert.AreEqual(new int[] { 2, 1, 2 }, data.Shape.Dimensions.ToArray());
}
}
public void TestMsgPackSamplerSlicing2()
{
const int NUM_SAMPLES = 300;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_SAMPLES; ++i)
{
var a = DataSourceFactory.Create(new float[] { i, i, i }, new int[] { 1, 1, 3 });
var dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(file, 5, false, 7, 10, false, 100))
{
int count = 0;
for (var i = 0; i < 10; ++i)
{
for (var j = 0; j < NUM_SAMPLES * 3 / 5; ++j)
{
var batch = sampler.GetNextMinibatch();
Assert.AreEqual(1, batch.Features.Count);
Assert.IsTrue(batch.Features.ContainsKey("a"));
var value = batch["a"];
CollectionAssert.AreEqual(new int[] { 1, 1, 5 }, value.Shape.Dimensions.ToArray());
var ds = DataSourceFactory.FromValue(value).ToArray();
Debug.WriteLine(string.Join(", ", ds));
CollectionAssert.AreEqual(new float[]
{
j * 5 / 3,
(j * 5 + 1) / 3,
(j * 5 + 2) / 3,
(j * 5 + 3) / 3,
(j * 5 + 4) / 3
}, ds);
count += 5;
}
}
}
}
public void TestWithoutParameter()
{
var exp = new Constant(new int[] { 2, 3 }, DataType.Float, 0) + Constant.Scalar(DataType.Float, 27);
var sampler = new ExpressionSampler("value", exp, null);
var batch = sampler.GetNextMinibatch();
Assert.AreEqual(batch.Features.Count, 1);
var data = batch["value"];
CollectionAssert.AreEqual(data.Shape.Dimensions.ToArray(), new int[] { 2, 3 });
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(ds.TypedData, new float[] { 27, 27, 27, 27, 27, 27 });
}
public void TestMsgPackSamplerRandomize()
{
const int NUM_CHUNKS = 30;
const int CHUNK_SIZE = 6;
const int MINIBATCH_SIZE = 2;
var data = new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
int FEATURE_DIM = data.Length / CHUNK_SIZE;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_CHUNKS; ++i)
{
var a = DataSourceFactory.Create(data, new int[] { FEATURE_DIM, CHUNK_SIZE });
var dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(file, MINIBATCH_SIZE, true, 10, 100, false, 100))
{
for (var i = 0; i < NUM_CHUNKS; ++i)
{
var values = new float[data.Length];
for (var j = 0; j < data.Length; j += FEATURE_DIM * MINIBATCH_SIZE)
{
var batch = sampler.GetNextMinibatch();
var value = DataSourceFactory.FromValue(batch["a"]);
CollectionAssert.AreEqual(new int[] { FEATURE_DIM, MINIBATCH_SIZE }, value.Shape.Dimensions.ToArray());
for (var k = 0; k < FEATURE_DIM * MINIBATCH_SIZE; ++k)
{
values[j + k] = value[k];
}
}
CollectionAssert.AreNotEqual(data, values);
var sorted = values.ToList();
sorted.Sort();
CollectionAssert.AreEqual(data, sorted);
}
}
}
public void TestConvolutionTranspose()
{
// input:
// | 0 0 0 0 0 |
// | 1 3 | strides = 2, 2 | 0 1 0 3 0 |
// | 2 4 | ================> | 0 0 0 0 0 |
// | 0 2 0 4 0 |
// | 0 0 0 0 0 |
//
// kernel:
// | 1 1 | transpose | 0 0 |
// | 0 0 | ================> | 1 1 |
//
// output:
// | 1 1 3 |
// | 0 0 0 |
// | 2 2 4 |
var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4 }, new int[] { 2, 2, 1 });
var convData = DataSourceFactory.Create(new float[] { 1, 0, 1, 0 }, new int[] { 2, 2, 1, 1 });
var input = CNTKLib.InputVariable(new int[] { 2, 2, 1 }, DataType.Float);
var convolutionMap = new Parameter(convData.ToNDArrayView());
var f = CNTKLib.ConvolutionTranspose(convolutionMap, input, new int[] { 2, 2, 1 }, new BoolVector(new bool[] { true }), new BoolVector(new bool[] { true }), new int[] { 0 }, new int[] { 1 }, 1, 0, "");
var inputs = new Dictionary <Variable, Value>()
{
{ input, inputData.ToValue() }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f.Output, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.UseDefaultDevice());
var result = DataSourceFactory.FromValue(outputs[f.Output]);
CollectionAssert.AreEqual(new int[] { 3, 3, 1, 1, 1 }, result.Shape.Dimensions);
Assert.AreEqual(1, result.Data[0]);
Assert.AreEqual(0, result.Data[1]);
Assert.AreEqual(2, result.Data[2]);
Assert.AreEqual(1, result.Data[3]);
}
public void TestConvolution()
{
// input:
// <ch. 1> <ch. 2>
// | 1 3 | | 1 3 |
// | 2 4 | | 2 4 |
//
// kernel:
// <ch. 1> <ch. 2>
// | 1 3 | | 1 3 |
// | 2 4 | | 2 4 |
//
// output:
// <ch. 1> <ch. 2>
// | 20 14 | = | (1 + 2 + 3 + 4) (3 + 4) | + | (1 + 2 + 3 + 4) (3 + 4) |
// | 12 8 | = | (2 + 4) 4 | + | (2 + 4) 4 |
var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 1, 2, 3, 4 }, new int[] { 2, 2, 2 });
var kernelData = DataSourceFactory.Create(new float[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, new int[] { 2, 2, 2, 4 });
var input = CNTKLib.InputVariable(new int[] { 2, 2, 2 }, DataType.Float);
var convolutionMap = new Parameter(kernelData.ToNDArrayView());
var f = CNTKLib.Convolution(convolutionMap, input, new int[] { 1, 1, 2 }, new BoolVector(new bool[] { true }), new BoolVector(new bool[] { true }), new int[] { 1 }, 1, 1);
var inputs = new Dictionary <Variable, Value>()
{
{ input, inputData.ToValue() }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f.Output, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.UseDefaultDevice());
var result = DataSourceFactory.FromValue(outputs[f.Output]);
CollectionAssert.AreEqual(new int[] { 2, 2, 4, 1, 1 }, result.Shape.Dimensions);
Assert.AreEqual(20, result.Data[0]);
Assert.AreEqual(12, result.Data[1]);
Assert.AreEqual(14, result.Data[2]);
Assert.AreEqual(8, result.Data[3]);
}
public void TestDataIngegrityAfterGarbageCollection()
{
for (var i = 0; i < 100; ++i)
{
var data = new float[] { 1, 2, 3, 4, 5, 6 };
MinibatchData m;
IntPtr sharedPtrAddress;
IntPtr valueAddress;
int c1;
unsafe
{
fixed (float* d = data)
{
var a = new NDArrayView(new int[] { 3, 2 }, data, DeviceDescriptor.CPUDevice);
var a2 = a.DeepClone();
var value = new Value(a2);
c1 = SwigMethods.GetSharedPtrUseCount(value);
sharedPtrAddress = SwigMethods.GetSwigPointerAddress(value);
valueAddress = SwigMethods.GetSharedPtrElementPointer(value);
m = new MinibatchData(value);
}
}
var c2 = SwigMethods.GetSharedPtrUseCount(m.data);
var sharedPtrAddress2 = SwigMethods.GetSwigPointerAddress(m.data);
var valueAddress2 = SwigMethods.GetSharedPtrElementPointer(m.data);
var c3 = SwigMethods.GetSharedPtrUseCount(m.data);
GC.Collect();
GC.Collect();
GC.Collect();
var c4 = SwigMethods.GetSharedPtrUseCount(m.data);
var sharedPtrAddress3 = SwigMethods.GetSwigPointerAddress(m.data);
var valueAddress3 = SwigMethods.GetSharedPtrElementPointer(m.data);
var ds = DataSourceFactory.FromValue(m.data);
Assert.AreEqual(6, ds.Data.Count);
CollectionAssert.AreEqual(new int[] { 3, 2 }, ds.Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 1, 2, 3, 4, 5, 6 }, ds.TypedData);
}
}
public void TestGetNextBatch()
{
var file = @"..\..\TestData\CTFTest1.txt";
var sampler = new CTFSampler(file, 1, false);
var m = sampler.GetNextMinibatch();
Assert.AreEqual(2, m.Features.Count);
Assert.AreEqual(1, m.SampleCount);
CollectionAssert.AreEqual(new int[] { 5, 1, 1 }, m.Features["f1"].Shape.Dimensions.ToArray());
CollectionAssert.AreEqual(new float[] { 1, 2, 3, 4, 5 }, DataSourceFactory.FromValue(m.Features["f1"]).ToArray());
m = sampler.GetNextMinibatch();
CollectionAssert.AreEqual(new int[] { 3, 1, 1 }, m.Features["f2"].Shape.Dimensions.ToArray());
CollectionAssert.AreEqual(new float[] { 6, 5, 4 }, DataSourceFactory.FromValue(m.Features["f2"]).ToArray());
}
public void TestSequence()
{
var features = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7 }, new int[] { 2, 2, -1 });
var ds = new Dictionary <string, IDataSource <float> >()
{
{ "input", features }
};
var sampler = new DataSourceSampler(ds, 2, false, true);
var batch = sampler.GetNextMinibatch();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(true, batch.SweepEnd);
var data = batch.Features["input"];
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 3, 4, 5, 6, 7 }, DataSourceFactory.FromValue(data).TypedData);
CollectionAssert.AreEqual(new int[] { 2, 2, 2 }, data.Shape.Dimensions.ToArray());
}
public void TestDataIngegrityAfterGarbageCollection()
{
for (var i = 0; i < 100; ++i)
{
NDArrayView a2;
using (var a = new NDArrayView(new int[] { 3, 2 }, new float[] { 1, 2, 3, 4, 5, 6 }, DeviceDescriptor.CPUDevice))
{
a2 = a.DeepClone();
}
GC.Collect();
var value = new Value(a2);
GC.Collect();
var ds = DataSourceFactory.FromValue(value);
Assert.AreEqual(6, ds.Data.Count);
CollectionAssert.AreEqual(new int[] { 3, 2 }, ds.Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 1, 2, 3, 4, 5, 6 }, ds.TypedData);
}
}
public void TestMsgPackSampler()
{
const int NUM_SAMPLES = 1000;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_SAMPLES; ++i)
{
var a = DataSourceFactory.Create(new float[] { i, i * 10, i * 100 }, new int[] { 3, 1, 1 });
var dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(file, 1, false, 3, 10, false, 100))
{
for (var i = 0; i < 10; ++i)
{
for (var j = 0; j < NUM_SAMPLES; ++j)
{
var batch = sampler.GetNextMinibatch();
Assert.AreEqual(1, batch.Features.Count);
Assert.IsTrue(batch.Features.ContainsKey("a"));
Assert.AreEqual((i * NUM_SAMPLES + j + 1) % 3 == 0, batch.SweepEnd);
var value = batch["a"];
CollectionAssert.AreEqual(new int[] { 3, 1, 1 }, value.Shape.Dimensions.ToArray());
var ds = DataSourceFactory.FromValue(value).ToArray();
// Debug.WriteLine(string.Join(", ", ds));
CollectionAssert.AreEqual(new float[] { j, j * 10, j * 100 }, ds);
}
}
}
}
