public void RowSparsePull(string key, NDArrayList @out, int priority = 0, NDArrayList row_ids = null)
{
if (@out == null)
{
throw new ArgumentNullException("@out");
}
if (row_ids == null)
{
throw new ArgumentNullException("row_ids");
}
var first_out = new NDArrayList();
if (row_ids.Length == 1)
{
first_out.Add(@out[0]);
}
else
{
first_out = @out.ToList();
}
NativeMethods.MXKVStorePullRowSparseEx(handle, 1, new[] { key },
MxUtil.GetNDArrayHandles(first_out.ToArray()), MxUtil.GetNDArrayHandles(row_ids), priority);
}
public static void Run()
{
// Create
var trainX = new NDArray(new float[] { 0, 0, 0, 1, 1, 0, 1, 1 }).Reshape(4, 2);
var trainY = new NDArray(new float[] { 0, 1, 1, 0 });
var batch_size = 2;
var train_data = new NDArrayIter(trainX, trainY, batch_size);
var val_data = new NDArrayIter(trainX, trainY, batch_size);
var net = new Sequential();
net.Add(new Dense(64, ActivationType.Relu));
net.Add(new Dense(1));
var gpus = TestUtils.ListGpus();
var ctxList = gpus.Count > 0 ? gpus.Select(x => Context.Gpu(x)).ToArray() : new[] { Context.Cpu() };
net.Initialize(new Uniform(), ctxList.ToArray());
var trainer = new Trainer(net.CollectParams(), new Adam());
var epoch = 1000;
var metric = new BinaryAccuracy();
var binary_crossentropy = new LogisticLoss();
float lossVal = 0;
for (var iter = 0; iter < epoch; iter++)
{
train_data.Reset();
lossVal = 0;
while (!train_data.End())
{
var batch = train_data.Next();
var data = Utils.SplitAndLoad(batch.Data[0], ctxList);
var label = Utils.SplitAndLoad(batch.Label[0], ctxList);
NDArrayList outputs = null;
using (var ag = Autograd.Record())
{
outputs = Enumerable.Zip(data, label, (x, y) =>
{
var z = net.Call(x);
NDArray loss = binary_crossentropy.Call(z, y);
loss.Backward();
lossVal += loss.Mean();
return(z);
}).ToList();
}
metric.Update(label, outputs.ToArray());
trainer.Step(batch.Data[0].Shape[0]);
}
var(name, acc) = metric.Get();
metric.Reset();
Console.WriteLine($"Loss: {lossVal}");
Console.WriteLine($"Training acc at epoch {iter}: {name}={acc * 100}%");
}
}
public static NDArrayList SplitData(NDArray data, int num_slice, int batch_axis = 0, bool even_split = true)
{
var size = data.Shape[batch_axis];
if (even_split && size % num_slice != 0)
{
throw new ArgumentException(string.Format(
"data with shape {0} cannot be evenly split into {1} slices along axis {2}. " +
"Use a batch size that's multiple of {3} or set even_split=False to allow " +
"uneven partitioning of data.", data.Shape, num_slice, batch_axis, num_slice));
}
var step = (int)Math.Truncate((double)size / num_slice);
if (!even_split && size < num_slice)
{
step = 1;
num_slice = size;
}
var slices = new NDArrayList();
if (batch_axis == 0)
{
for (var i = 0; i < num_slice; i++)
{
if (i < num_slice)
{
slices.Add(data[string.Format("{0}:{1}", i * step, (i + 1) * step)]);
}
}
}
else if (even_split)
{
slices.Add(nd.Split(data, num_slice, batch_axis));
}
else
{
for (var i = 0; i < num_slice; i++)
{
if (i < num_slice - 1)
{
slices.Add(data[string.Format("{0}:{1}", i * step, (i + 1) * step)]);
}
else
{
slices.Add(nd.SliceAxis(data, batch_axis, i * step, (i + 1) * step));
}
}
}
return(slices.ToArray());
}
public static NDArrayList SplitAndLoad(NDArray data, Context[] ctx_list, int batch_axis = 0,
bool even_split = true)
{
if (ctx_list.Length == 1)
{
return(data.AsInContext(ctx_list[0]));
}
var slices = SplitData(data, ctx_list.Length, batch_axis, even_split);
var result = new NDArrayList();
result = slices.Zip(ctx_list, (i, ctx) => { return(i.AsInContext(ctx)); }).ToList();
return(result.ToArray());
}
private NDArrayList _concat(NDArrayList first_data, NDArrayList second_data)
{
if (first_data.Length != second_data.Length)
{
throw new Exception("Data source should be of same size.");
}
var result = new NDArrayList();
for (var i = 0; i < first_data.Length; i++)
{
result.Add(
nd.Concat(new NDArrayList(first_data[i], second_data[i]), 0)
);
}
return(result.ToArray());
}
private NDArrayList _getdata(NDArrayDict data_source, int?start = null, int?end = null)
{
if (!start.HasValue && !end.HasValue)
{
throw new ArgumentException("Should atleast specify start or end");
}
start = start.HasValue ? start : 0;
end = end.HasValue ? end : data_source.First().Value.Shape[0];
var result = new NDArrayList();
foreach (var x in data_source)
{
result.Add(x.Value.Slice(start.Value, end));
}
return(result.ToArray());
}
public IEnumerable <(NDArrayList, int, DataBatch)> IterPredict(DataIter eval_data, int?num_batch = null,
bool reset = true, int epoch = 0, Func <DataBatch, NDArrayDict> sparse_row_id_fn = null)
{
if (!Binded && !ParamsInitialized)
{
throw new Exception("Module not binded and param initialized");
}
if (reset)
{
eval_data.Reset();
}
while (eval_data.End())
{
if (num_batch.HasValue && eval_data.Cursor == num_batch.Value)
{
break;
}
var eval_batch = eval_data.Next();
Prepare(eval_batch, sparse_row_id_fn);
Forward(eval_batch, false);
var pad = eval_batch.Pad.Value;
var outputs = new NDArrayList();
foreach (var list in GetOutputs())
{
foreach (var @out in list)
{
outputs.Add(@out[$"0:{@out.Shape[0] - pad}"]);
}
}
yield return(outputs.ToArray(), eval_data.Cursor, eval_batch);
}
}
public static void RunSimple()
{
var mnist = TestUtils.GetMNIST(); //Get the MNIST dataset, it will download if not found
var batch_size = 100; //Set training batch size
var train_data = new NDArrayIter(mnist["train_data"], mnist["train_label"], batch_size);
var val_data = new NDArrayIter(mnist["test_data"], mnist["test_label"], batch_size);
// Define simple network with dense layers
var net = new Sequential();
net.Add(new Dense(128, ActivationType.Relu));
net.Add(new Dense(64, ActivationType.Relu));
net.Add(new Dense(10));
//Set context, multi-gpu supported
var gpus = TestUtils.ListGpus();
var ctx = gpus.Count > 0 ? gpus.Select(x => Context.Gpu(x)).ToArray() : new[] { Context.Cpu(0) };
//Initialize the weights
net.Initialize(new Xavier(magnitude: 2.24f), ctx);
//Create the trainer with all the network parameters and set the optimizer
var trainer = new Trainer(net.CollectParams(), new Adam());
var epoch = 10;
var metric = new Accuracy(); //Use Accuracy as the evaluation metric.
var softmax_cross_entropy_loss = new SoftmaxCrossEntropyLoss();
float lossVal = 0; //For loss calculation
for (var iter = 0; iter < epoch; iter++)
{
var tic = DateTime.Now;
// Reset the train data iterator.
train_data.Reset();
lossVal = 0;
// Loop over the train data iterator.
while (!train_data.End())
{
var batch = train_data.Next();
// Splits train data into multiple slices along batch_axis
// and copy each slice into a context.
var data = Utils.SplitAndLoad(batch.Data[0], ctx, batch_axis: 0);
// Splits train labels into multiple slices along batch_axis
// and copy each slice into a context.
var label = Utils.SplitAndLoad(batch.Label[0], ctx, batch_axis: 0);
var outputs = new NDArrayList();
// Inside training scope
NDArray loss = null;
for (int i = 0; i < data.Length; i++)
{
using (var ag = Autograd.Record())
{
var x = data[i];
var y = label[i];
var z = net.Call(x);
// Computes softmax cross entropy loss.
loss = softmax_cross_entropy_loss.Call(z, y);
outputs.Add(z);
}
// Backpropagate the error for one iteration.
loss.Backward();
lossVal += loss.Mean();
}
// Updates internal evaluation
metric.Update(label, outputs.ToArray());
// Make one step of parameter update. Trainer needs to know the
// batch size of data to normalize the gradient by 1/batch_size.
trainer.Step(batch.Data[0].Shape[0]);
}
var toc = DateTime.Now;
// Gets the evaluation result.
var(name, acc) = metric.Get();
// Reset evaluation result to initial state.
metric.Reset();
Console.Write($"Loss: {lossVal} ");
Console.WriteLine($"Training acc at epoch {iter}: {name}={(acc * 100).ToString("0.##")}%, Duration: {(toc - tic).TotalSeconds.ToString("0.#")}s");
}
}
public static void RunConv()
{
var mnist = TestUtils.GetMNIST();
var batch_size = 128;
var train_data = new NDArrayIter(mnist["train_data"], mnist["train_label"], batch_size, true);
var val_data = new NDArrayIter(mnist["test_data"], mnist["test_label"], batch_size);
var net = new Sequential();
net.Add(new Conv2D(20, kernel_size: (5, 5), activation: ActivationType.Tanh));
net.Add(new MaxPool2D(pool_size: (2, 2), strides: (2, 2)));
net.Add(new Conv2D(50, kernel_size: (5, 5), activation: ActivationType.Tanh));
net.Add(new MaxPool2D(pool_size: (2, 2), strides: (2, 2)));
net.Add(new Flatten());
net.Add(new Dense(500, ActivationType.Tanh));
net.Add(new Dense(10));
var gpus = TestUtils.ListGpus();
var ctx = gpus.Count > 0 ? gpus.Select(x => Context.Gpu(x)).ToArray() : new[] { Context.Cpu(0) };
net.Initialize(new Xavier(magnitude: 2.24f), ctx);
var trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.02f));
var epoch = 10;
var metric = new Accuracy();
var softmax_cross_entropy_loss = new SoftmaxCELoss();
float lossVal = 0;
for (var iter = 0; iter < epoch; iter++)
{
var tic = DateTime.Now;
train_data.Reset();
lossVal = 0;
while (!train_data.End())
{
var batch = train_data.Next();
var data = Utils.SplitAndLoad(batch.Data[0], ctx, batch_axis: 0);
var label = Utils.SplitAndLoad(batch.Label[0], ctx, batch_axis: 0);
var outputs = new NDArrayList();
using (var ag = Autograd.Record())
{
for (var i = 0; i < data.Length; i++)
{
var x = data[i];
var y = label[i];
var z = net.Call(x);
NDArray loss = softmax_cross_entropy_loss.Call(z, y);
loss.Backward();
lossVal += loss.Mean();
outputs.Add(z);
}
//outputs = Enumerable.Zip(data, label, (x, y) =>
//{
// var z = net.Call(x);
// NDArray loss = softmax_cross_entropy_loss.Call(z, y);
// loss.Backward();
// lossVal += loss.Mean();
// return z;
//}).ToList();
}
metric.Update(label, outputs.ToArray());
trainer.Step(batch.Data[0].Shape[0]);
}
var toc = DateTime.Now;
var(name, acc) = metric.Get();
metric.Reset();
Console.Write($"Loss: {lossVal} ");
Console.WriteLine($"Training acc at epoch {iter}: {name}={(acc * 100).ToString("0.##")}%, Duration: {(toc - tic).TotalSeconds.ToString("0.#")}s");
}
}
public List <NDArrayList> Predict(DataIter eval_data, int?num_batch = null, bool merge_batches = true,
bool reset = true, bool always_output_list = true, Func <DataBatch, NDArrayDict> sparse_row_id_fn = null)
{
if (!Binded && !ParamsInitialized)
{
throw new Exception("Module not binded and param initialized");
}
if (reset)
{
eval_data.Reset();
}
var output_list = new List <NDArrayList>();
var output_list2 = new NDArrayList();
while (eval_data.End())
{
if (num_batch.HasValue && eval_data.Cursor == num_batch.Value)
{
break;
}
var eval_batch = eval_data.Next();
Prepare(eval_batch, sparse_row_id_fn);
Forward(eval_batch, false);
var pad = eval_batch.Pad.Value;
var outputs = new NDArrayList();
foreach (var list in GetOutputs())
{
foreach (var @out in list)
{
outputs.Add(@out[$"0:{@out.Shape[0] - pad}"].Copy());
}
}
output_list.Add(outputs.ToArray());
}
if (output_list.Count == 0)
{
return(output_list);
}
if (merge_batches)
{
var num_outputs = output_list[0].Length;
foreach (var @out in output_list)
{
if (@out.Length != num_outputs)
{
throw new Exception("Cannot merge batches, as num of outputs is not the same " +
"in mini-batches. Maybe bucketing is used?");
}
output_list2.Add(nd.Concat(@out));
}
return(new List <NDArrayList> {
output_list2.ToArray()
});
}
return(output_list);
}
