void evaluate_network()
{
var batch_size = 128;
var pos = 0;
var accuracy = 0.0;
var num_batches = 0;
var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
while (pos < test_images.Length)
{
var pos_end = Math.Min(pos + batch_size, test_images.Length);
var minibatch_images = Util.get_tensors(image_tensor.Shape, test_images, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(label_tensor.Shape, test_labels, pos, pos_end, computeDevice);
var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr()
{
{ image_tensor, minibatch_images }, { label_tensor, minibatch_labels }
};
var minibatch_accuracy = evaluator.TestMinibatch(feed_dictionary, computeDevice);
accuracy += minibatch_accuracy;
pos = pos_end;
num_batches++;
}
accuracy /= num_batches;
Console.WriteLine(string.Format("Accuracy:{0:F3}", accuracy));
}
void train_network()
{
int epochs = 5;
int batch_size = 64;
for (int current_epoch = 0; current_epoch < epochs; current_epoch++)
{
// training phase
var train_indices = Util.shuffled_indices(60000);
var pos = 0;
while (pos < train_indices.Length)
{
var pos_end = Math.Min(pos + batch_size, train_indices.Length);
var minibatch_images = Util.get_tensors(image_tensor.Shape, train_images, train_indices, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(label_tensor.Shape, train_labels, train_indices, pos, pos_end, computeDevice);
var feed_dictionary = new feed_t()
{
{ image_tensor, minibatch_images }, { label_tensor, minibatch_labels }
};
trainer.TrainMinibatch(feed_dictionary, false, computeDevice);
pos = pos_end;
}
// evaluation phase
var accuracy = 0.0;
var num_batches = 0;
pos = 0;
while (pos < test_images.Length)
{
var pos_end = Math.Min(pos + batch_size, test_images.Length);
var minibatch_images = Util.get_tensors(image_tensor.Shape, test_images, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(label_tensor.Shape, test_labels, pos, pos_end, computeDevice);
var feed_dictionary = new test_feed_t()
{
{ image_tensor, minibatch_images }, { label_tensor, minibatch_labels }
};
var minibatch_accuracy = evaluator.TestMinibatch(feed_dictionary, computeDevice);
accuracy += minibatch_accuracy;
pos = pos_end;
num_batches++;
}
accuracy /= num_batches;
Console.WriteLine(string.Format("Epoch {0}/{1}, Test accuracy:{2:F3}", current_epoch + 1, epochs, 1.0 - accuracy));
}
}
double evaluation_phase(int[] validation_indices)
{
var pos = 0;
var num_batches = 0;
var epoch_evaluation_accuracy = 0.0;
while (pos < validation_indices.Length)
{
var pos_end = Math.Min(pos + batch_size, validation_indices.Length);
var minibatch_x = Util.get_tensors(x_tensor.Shape, x_train, validation_indices, pos, pos_end, computeDevice);
var minibatch_y = Util.get_tensors(y_tensor.Shape, y_train, validation_indices, pos, pos_end, computeDevice);
var feed_dictionary = new test_feed_t()
{
{ x_tensor, minibatch_x }, { y_tensor, minibatch_y }
};
var minibatch_accuracy = evaluator.TestMinibatch(feed_dictionary, computeDevice);
epoch_evaluation_accuracy += minibatch_accuracy;
num_batches++;
pos = pos_end;
}
epoch_evaluation_accuracy /= num_batches;
return(epoch_evaluation_accuracy);
}
List <List <double> > train_with_extracted_features()
{
float[][] train_features = null, validation_features = null, test_features = null;
float[][] train_labels = null, validation_labels = null, test_labels = null;
compute_features_and_labels(ref train_features, ref validation_features, ref test_features, ref train_labels, ref validation_labels, ref test_labels);
var labels = CNTK.Variable.InputVariable(new int[] { 2, }, CNTK.DataType.Float, "labels_var");
var features = CNTK.Variable.InputVariable(new int[] { extracted_feature_length, }, CNTK.DataType.Float, "features_var");
var model = CNTK.CNTKLib.ReLU(Util.Dense(features, 256, computeDevice));
model = CNTK.CNTKLib.Dropout(model, 0.5);
model = Util.Dense(model, 2, computeDevice);
var loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(model.Output, labels);
var accuracy_function = CNTK.CNTKLib.ClassificationError(model.Output, labels);
var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.0001, 1), new CNTK.TrainingParameterScheduleDouble(0.99, 1));
var trainer = CNTK.Trainer.CreateTrainer(model, loss_function, accuracy_function, new CNTK.Learner[] { learner });
var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
var training_accuracy = new List <double>();
var validation_accuracy = new List <double>();
for (int epoch = 0; epoch < max_epochs; epoch++)
{
// training phase
var epoch_training_error = 0.0;
var train_indices = Util.shuffled_indices(train_features.Length);
var pos = 0;
var num_batches = 0;
while (pos < train_indices.Length)
{
var pos_end = Math.Min(pos + batch_size, train_indices.Length);
var minibatch_images = Util.get_tensors(features.Shape, train_features, train_indices, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(labels.Shape, train_labels, train_indices, pos, pos_end, computeDevice);
var feed_dictionary = new feed_t()
{
{ features, minibatch_images }, { labels, minibatch_labels }
};
trainer.TrainMinibatch(feed_dictionary, false, computeDevice);
epoch_training_error += trainer.PreviousMinibatchEvaluationAverage();
num_batches++;
pos = pos_end;
}
epoch_training_error /= num_batches;
training_accuracy.Add(1.0 - epoch_training_error);
// evaluation phase
var epoch_validation_error = 0.0;
num_batches = 0;
pos = 0;
while (pos < validation_features.Length)
{
var pos_end = Math.Min(pos + batch_size, validation_features.Length);
var minibatch_images = Util.get_tensors(features.Shape, validation_features, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(labels.Shape, validation_labels, pos, pos_end, computeDevice);
var feed_dictionary = new test_feed_t()
{
{ features, minibatch_images }, { labels, minibatch_labels }
};
epoch_validation_error += evaluator.TestMinibatch(feed_dictionary, computeDevice);
pos = pos_end;
num_batches++;
}
epoch_validation_error /= num_batches;
validation_accuracy.Add(1.0 - epoch_validation_error);
Console.WriteLine($"Epoch {epoch + 1:D2}/{max_epochs}, training_accuracy={1.0-epoch_training_error:F3}, validation accuracy:{1-epoch_validation_error:F3}");
if (epoch_training_error < 0.001)
{
break;
}
}
return(new List <List <double> >()
{
training_accuracy, validation_accuracy
});
}
List <List <double> > fit_generator(bool sequence_mode, CNTK.Variable x, CNTK.Variable y, CNTK.Function model, CNTK.Trainer trainer, CNTK.Evaluator evaluator, GeneratorsInfo gi, int epochs, int steps_per_epoch, CNTK.DeviceDescriptor computeDevice)
{
var history = new List <List <double> >()
{
new List <double>(), new List <double>()
};
var train_enumerator = gi.train_gen.GetEnumerator();
var val_enumerator = gi.val_gen.GetEnumerator();
var x_minibatch_dims = new List <int>(x.Shape.Dimensions);
if (sequence_mode == false)
{
x_minibatch_dims.Add(gi.batch_size);
}
for (int current_epoch = 0; current_epoch < epochs; current_epoch++)
{
var epoch_start_time = DateTime.Now;
var epoch_training_error = 0.0;
{
var num_total_samples = 0;
for (int s = 0; s < steps_per_epoch; s++)
{
train_enumerator.MoveNext();
var st = train_enumerator.Current;
var x_minibatch = create_x_minibatch(sequence_mode, x, gi, st, computeDevice);
var y_minibatch = CNTK.Value.CreateBatch(y.Shape, st.targets, computeDevice);
var feed_dictionary = new Dictionary <CNTK.Variable, CNTK.Value> {
{ x, x_minibatch }, { y, y_minibatch }
};
bool isSweepEndInArguments = (s == (steps_per_epoch - 1));
trainer.TrainMinibatch(feed_dictionary, isSweepEndInArguments, computeDevice);
var minibatch_metric = trainer.PreviousMinibatchEvaluationAverage();
epoch_training_error += minibatch_metric * st.targets.Length;
num_total_samples += st.targets.Length;
x_minibatch.Erase();
y_minibatch.Erase();
}
epoch_training_error /= num_total_samples;
}
history[0].Add(epoch_training_error);
var epoch_validation_error = 0.0;
{
var num_total_samples = 0;
for (int s = 0; s < gi.val_steps; s++)
{
val_enumerator.MoveNext();
var st = val_enumerator.Current;
var x_minibatch = create_x_minibatch(sequence_mode, x, gi, st, computeDevice);
var y_minibatch = CNTK.Value.CreateBatch(y.Shape, st.targets, computeDevice);
var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr()
{
{ x, x_minibatch }, { y, y_minibatch }
};
var minibatch_metric = evaluator.TestMinibatch(feed_dictionary, computeDevice);
epoch_validation_error += minibatch_metric * st.targets.Length;
num_total_samples += st.targets.Length;
x_minibatch.Erase();
y_minibatch.Erase();
}
epoch_validation_error /= num_total_samples;
}
history[1].Add(epoch_validation_error);
var elapsedTime = DateTime.Now.Subtract(epoch_start_time);
Console.WriteLine($"Epoch {current_epoch + 1:D2}/{epochs}, Elapsed time: {elapsedTime.TotalSeconds:F3} seconds. " +
$"Training Error: {epoch_training_error:F3}. Validation Error: {epoch_validation_error:F3}.");
}
return(history);
}
void train()
{
if (System.IO.File.Exists(model_filename))
{
System.Console.WriteLine("Model exists: " + model_filename);
return;
}
create_model();
load_data();
var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.001), new CNTK.TrainingParameterScheduleDouble(0.9));
var trainer = CNTK.CNTKLib.CreateTrainer(model, loss, loss, new CNTK.LearnerVector()
{
learner
});
var evaluator = CNTK.CNTKLib.CreateEvaluator(loss);
var batch_size = 16;
var epochs = 10;
Console.WriteLine("Training on: " + computeDevice.AsString());
for (int current_epoch = 0; current_epoch < epochs; current_epoch++)
{
var epoch_start_time = DateTime.Now;
var epoch_training_loss = 0.0;
{
var train_indices = Util.shuffled_indices(x_train.Length);
var pos = 0;
while (pos < train_indices.Length)
{
var pos_end = Math.Min(pos + batch_size, train_indices.Length);
var x_minibatch = Util.get_tensors(input_img.Shape, x_train, train_indices, pos, pos_end, computeDevice);
var feed_dictionary = new Dictionary <CNTK.Variable, CNTK.Value> {
{ input_img, x_minibatch }
};
trainer.TrainMinibatch(feed_dictionary, false, computeDevice);
epoch_training_loss += (pos_end - pos) * trainer.PreviousMinibatchLossAverage();
pos = pos_end;
x_minibatch.Erase(); x_minibatch.Dispose(); x_minibatch = null;
}
epoch_training_loss /= x_train.Length;
}
var epoch_validation_loss = 0.0;
{
var pos = 0;
while (pos < x_test.Length)
{
var pos_end = Math.Min(pos + batch_size, x_test.Length);
var x_minibatch = Util.get_tensors(input_img.Shape, x_test, pos, pos_end, computeDevice);
var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr()
{
{ input_img, x_minibatch }
};
epoch_validation_loss += (pos_end - pos) * evaluator.TestMinibatch(feed_dictionary, computeDevice);
pos = pos_end;
x_minibatch.Erase(); x_minibatch.Dispose(); x_minibatch = null;
}
epoch_validation_loss /= x_test.Length;
}
var elapsedTime = DateTime.Now.Subtract(epoch_start_time);
Console.Write($"Epoch {current_epoch + 1:D2}/{epochs}, Elapsed time: {elapsedTime.TotalSeconds:F3} seconds. ");
Console.WriteLine($"Training loss: {epoch_training_loss:F3}. Validation Loss: {epoch_validation_loss:F3}.");
}
model.Save(model_filename);
}
