void create_network()
{
computeDevice = Util.get_compute_device();
features_tensor = CNTK.Variable.InputVariable(new int[] { image_height, image_width, num_channels }, CNTK.DataType.Float);
label_tensor = CNTK.Variable.InputVariable(new int[] { 2 }, CNTK.DataType.Float);
var scalar_factor = CNTK.Constant.Scalar <float>((float)(1.0 / 255.0), computeDevice);
network = CNTK.CNTKLib.ElementTimes(scalar_factor, features_tensor);
network = Util.Convolution2DWithReLU(network, 32, new int[] { 3, 3 }, computeDevice);
network = CNTK.CNTKLib.Pooling(network, CNTK.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 });
network = Util.Convolution2DWithReLU(network, 64, new int[] { 3, 3 }, computeDevice);
network = CNTK.CNTKLib.Pooling(network, CNTK.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 });
network = Util.Convolution2DWithReLU(network, 128, new int[] { 3, 3 }, computeDevice);
network = CNTK.CNTKLib.Pooling(network, CNTK.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 });
network = Util.Convolution2DWithReLU(network, 128, new int[] { 3, 3 }, computeDevice);
network = CNTK.CNTKLib.Pooling(network, CNTK.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 });
network = CNTK.CNTKLib.Dropout(network, 0.5);
network = CNTK.CNTKLib.ReLU(Util.Dense(network, 512, computeDevice));
network = Util.Dense(network, 2, computeDevice);
loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(network.Output, label_tensor);
accuracy_function = CNTK.CNTKLib.ClassificationError(network.Output, label_tensor);
var parameterVector = new CNTK.ParameterVector((System.Collections.ICollection)network.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(parameterVector, new CNTK.TrainingParameterScheduleDouble(0.0001, 1), new CNTK.TrainingParameterScheduleDouble(0.99, 1));
trainer = CNTK.CNTKLib.CreateTrainer(network, loss_function, accuracy_function, new CNTK.LearnerVector()
{
learner
});
evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
}
void create_network()
{
computeDevice = Util.get_compute_device();
Console.WriteLine("Compute Device: " + computeDevice.AsString());
image_tensor = CNTK.Variable.InputVariable(CNTK.NDShape.CreateNDShape(new int[] { 28, 28, 1 }), CNTK.DataType.Float);
label_tensor = CNTK.Variable.InputVariable(CNTK.NDShape.CreateNDShape(new int[] { 10 }), CNTK.DataType.Float);
network = image_tensor;
network = Util.Convolution2DWithReLU(network, 32, new int[] { 3, 3 }, computeDevice);
network = CNTK.CNTKLib.Pooling(network, CNTK.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 });
network = Util.Convolution2DWithReLU(network, 64, new int[] { 3, 3 }, computeDevice);
network = CNTK.CNTKLib.Pooling(network, CNTK.PoolingType.Max, new int[] { 2, 2 }, new int[] { 2 });
network = Util.Convolution2DWithReLU(network, 64, new int[] { 3, 3 }, computeDevice);
network = Util.Dense(network, 64, computeDevice);
network = CNTK.CNTKLib.ReLU(network);
network = Util.Dense(network, 10, computeDevice);
Util.log_number_of_parameters(network);
loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(network.Output, label_tensor);
accuracy_function = CNTK.CNTKLib.ClassificationError(network.Output, label_tensor);
var parameterVector = new CNTK.ParameterVector((System.Collections.ICollection)network.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(parameterVector, new CNTK.TrainingParameterScheduleDouble(0.001, 1), new CNTK.TrainingParameterScheduleDouble(0.99, 1));
trainer = CNTK.CNTKLib.CreateTrainer(network, loss_function, accuracy_function, new CNTK.LearnerVector()
{
learner
});
evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
}
/// <summary>
/// Get an RMSProp learner to train the network.
/// </summary>
/// <param name="input">The network to train.</param>
/// <param name="learningRateSchedule">The learning rate schedule.</param>
/// <param name="gamma">The gamma value.</param>
/// <param name="inc">The inc value.</param>
/// <param name="dec">The dec value.</param>
/// <param name="max">The max value.</param>
/// <param name="min">The min value.</param>
/// <returns>An RMSProp learner to train the network.</returns>
public static CNTK.Learner GetRMSPropLearner(
this CNTK.Function input,
double learningRateSchedule,
double gamma,
double inc,
double dec,
double max,
double min)
{
var parameterVector = new CNTK.ParameterVector((System.Collections.ICollection)input.Parameters());
return(CNTK.CNTKLib.RMSPropLearner(
parameterVector,
new CNTK.TrainingParameterScheduleDouble(learningRateSchedule),
gamma,
inc,
dec,
max,
min));
}
List <List <double> > train_mse_cntk(bool sequence_mode, CNTK.Variable x, CNTK.Variable y, CNTK.Function model, GeneratorsInfo gi, int epochs, int steps_per_epoch, CNTK.DeviceDescriptor computeDevice)
{
var loss_function = CNTK.CNTKLib.SquaredError(model, y);
var accuracy_function = loss_function;
var lr = 0.001;
var parameterVector = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(parameterVector,
new CNTK.TrainingParameterScheduleDouble(lr /*, (uint)batch_size*/),
new CNTK.TrainingParameterScheduleDouble(0.9 /*, (uint)batch_size*/),
unitGain: false);
var trainer = CNTK.CNTKLib.CreateTrainer(model, loss_function, accuracy_function, new CNTK.LearnerVector()
{
learner
});
var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
var history = fit_generator(sequence_mode, x, y, model, trainer, evaluator, gi, epochs, steps_per_epoch, computeDevice);
return(history);
}
void create_network()
{
computeDevice = Util.get_compute_device();
Console.WriteLine("Compute Device: " + computeDevice.AsString());
image_tensor = CNTK.Variable.InputVariable(CNTK.NDShape.CreateNDShape(new int[] { 28, 28 }), CNTK.DataType.Float);
label_tensor = CNTK.Variable.InputVariable(CNTK.NDShape.CreateNDShape(new int[] { 10 }), CNTK.DataType.Float);
network = Util.Dense(image_tensor, 512, computeDevice);
network = CNTK.CNTKLib.ReLU(network);
network = Util.Dense(network, 10, computeDevice);
loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(network.Output, label_tensor);
accuracy_function = CNTK.CNTKLib.ClassificationError(network.Output, label_tensor);
var parameterVector = new CNTK.ParameterVector((System.Collections.ICollection)network.Parameters());
var learner = CNTK.CNTKLib.RMSPropLearner(parameterVector, new CNTK.TrainingParameterScheduleDouble(0.99), 0.95, 2.0, 0.5, 2.0, 0.5);
trainer = CNTK.CNTKLib.CreateTrainer(network, loss_function, accuracy_function, new CNTK.LearnerVector()
{
learner
});
}
void train(CNTK.Function model, float[][] labels)
{
var content_and_style_outputs = traverse_content_and_styles_nodes(model);
var label_variables = new List <CNTK.Variable>();
for (int i = 0; i < labels.Length; i++)
{
var shape = content_and_style_outputs[i].Shape;
var input_variable = CNTK.Variable.InputVariable(shape, CNTK.DataType.Float, "content_and_style_" + i);
label_variables.Add(input_variable);
}
var loss_function = create_loss_function(model, content_and_style_outputs, label_variables);
var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(10), new CNTK.TrainingParameterScheduleDouble(0.95));
var trainer = CNTK.CNTKLib.CreateTrainer(model, loss_function, loss_function, new CNTK.LearnerVector()
{
learner
});
var batch = create_batch(loss_function, labels);
Console.WriteLine("Training on a " + computeDevice.AsString());
var startTime = DateTime.Now;
for (int i = 0; i < 301; i++)
{
trainer.TrainMinibatch(batch, true, computeDevice);
if (i % 100 == 0)
{
Console.WriteLine($"epoch {i}, loss={trainer.PreviousMinibatchLossAverage():F3}");
}
}
var elapsedTime = DateTime.Now.Subtract(startTime);
Console.WriteLine($"Done in {elapsedTime.TotalSeconds:F1} seconds");
}
void create_network()
{
computeDevice = Util.get_compute_device();
Console.WriteLine("Compute Device: " + computeDevice.AsString());
x_tensor = CNTK.Variable.InputVariable(CNTK.NDShape.CreateNDShape(new int[] { 13 }), CNTK.DataType.Float);
y_tensor = CNTK.Variable.InputVariable(CNTK.NDShape.CreateNDShape(new int[] { 1 }), CNTK.DataType.Float);
network = CNTK.CNTKLib.ReLU(Util.Dense(x_tensor, 64, computeDevice));
network = CNTK.CNTKLib.ReLU(Util.Dense(network, 64, computeDevice));
network = Util.Dense(network, 1, computeDevice);
loss_function = Util.MeanSquaredError(network.Output, y_tensor);
accuracy_function = Util.MeanAbsoluteError(network.Output, y_tensor);
var parameterVector = new CNTK.ParameterVector((System.Collections.ICollection)network.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(parameterVector, new CNTK.TrainingParameterScheduleDouble(0.001, 1), new CNTK.TrainingParameterScheduleDouble(0.9, 1), true);
trainer = CNTK.CNTKLib.CreateTrainer(network, loss_function, accuracy_function, new CNTK.LearnerVector()
{
learner
});
evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
}
List <List <double> > train_with_augmentation(bool use_finetuning)
{
var labels = CNTK.Variable.InputVariable(new int[] { 2 }, CNTK.DataType.Float, "labels");
var features = CNTK.Variable.InputVariable(new int[] { 150, 150, 3 }, CNTK.DataType.Float, "features");
var scalar_factor = CNTK.Constant.Scalar <float>((float)(1.0 / 255.0), computeDevice);
var scaled_features = CNTK.CNTKLib.ElementTimes(scalar_factor, features);
var conv_base = VGG16.get_model(scaled_features, computeDevice, use_finetuning);
var model = Util.Dense(conv_base, 256, computeDevice);
model = CNTK.CNTKLib.ReLU(model);
model = CNTK.CNTKLib.Dropout(model, 0.5);
model = Util.Dense(model, 2, computeDevice);
var loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(model.Output, labels);
var accuracy_function = CNTK.CNTKLib.ClassificationError(model.Output, labels);
var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.0001, 1), new CNTK.TrainingParameterScheduleDouble(0.99, 1));
var trainer = CNTK.Trainer.CreateTrainer(model, loss_function, accuracy_function, new CNTK.Learner[] { learner });
var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
var train_minibatch_source = create_minibatch_source(features.Shape, 0, 1000, "train", is_training: true, use_augmentations: true);
var validation_minibatch_source = create_minibatch_source(features.Shape, 1000, 500, "validation", is_training: false, use_augmentations: false);
var train_featuresStreamInformation = train_minibatch_source.StreamInfo("features");
var train_labelsStreamInformation = train_minibatch_source.StreamInfo("labels");
var validation_featuresStreamInformation = validation_minibatch_source.StreamInfo("features");
var validation_labelsStreamInformation = validation_minibatch_source.StreamInfo("labels");
var training_accuracy = new List <double>();
var validation_accuracy = new List <double>();
for (int epoch = 0; epoch < max_epochs; epoch++)
{
var startTime = DateTime.Now;
// training phase
var epoch_training_error = 0.0;
var pos = 0;
var num_batches = 0;
while (pos < 2000)
{
var pos_end = Math.Min(pos + batch_size, 2000);
var minibatch_data = train_minibatch_source.GetNextMinibatch((uint)(pos_end - pos), computeDevice);
var feed_dictionary = new batch_t()
{
{ features, minibatch_data[train_featuresStreamInformation] },
{ labels, minibatch_data[train_labelsStreamInformation] }
};
trainer.TrainMinibatch(feed_dictionary, computeDevice);
epoch_training_error += trainer.PreviousMinibatchEvaluationAverage();
num_batches++;
pos = pos_end;
}
epoch_training_error /= num_batches;
training_accuracy.Add(1.0 - epoch_training_error);
// evaluation phase
var epoch_validation_error = 0.0;
num_batches = 0;
pos = 0;
while (pos < 1000)
{
var pos_end = Math.Min(pos + batch_size, 1000);
var minibatch_data = validation_minibatch_source.GetNextMinibatch((uint)(pos_end - pos), computeDevice);
var feed_dictionary = new CNTK.UnorderedMapVariableMinibatchData()
{
{ features, minibatch_data[validation_featuresStreamInformation] },
{ labels, minibatch_data[validation_labelsStreamInformation] }
};
epoch_validation_error += evaluator.TestMinibatch(feed_dictionary);
pos = pos_end;
num_batches++;
}
epoch_validation_error /= num_batches;
validation_accuracy.Add(1.0 - epoch_validation_error);
var elapsedTime = DateTime.Now.Subtract(startTime);
Console.WriteLine($"Epoch {epoch + 1:D2}/{max_epochs}, training_accuracy={1.0 - epoch_training_error:F3}, validation accuracy:{1 - epoch_validation_error:F3}, elapsed time={elapsedTime.TotalSeconds:F1} seconds");
if (epoch_training_error < 0.001)
{
break;
}
}
return(new List <List <double> >()
{
training_accuracy, validation_accuracy
});
}
List <List <double> > train_with_extracted_features()
{
float[][] train_features = null, validation_features = null, test_features = null;
float[][] train_labels = null, validation_labels = null, test_labels = null;
compute_features_and_labels(ref train_features, ref validation_features, ref test_features, ref train_labels, ref validation_labels, ref test_labels);
var labels = CNTK.Variable.InputVariable(new int[] { 2, }, CNTK.DataType.Float, "labels_var");
var features = CNTK.Variable.InputVariable(new int[] { extracted_feature_length, }, CNTK.DataType.Float, "features_var");
var model = CNTK.CNTKLib.ReLU(Util.Dense(features, 256, computeDevice));
model = CNTK.CNTKLib.Dropout(model, 0.5);
model = Util.Dense(model, 2, computeDevice);
var loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(model.Output, labels);
var accuracy_function = CNTK.CNTKLib.ClassificationError(model.Output, labels);
var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.0001, 1), new CNTK.TrainingParameterScheduleDouble(0.99, 1));
var trainer = CNTK.Trainer.CreateTrainer(model, loss_function, accuracy_function, new CNTK.Learner[] { learner });
var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function);
var training_accuracy = new List <double>();
var validation_accuracy = new List <double>();
for (int epoch = 0; epoch < max_epochs; epoch++)
{
// training phase
var epoch_training_error = 0.0;
var train_indices = Util.shuffled_indices(train_features.Length);
var pos = 0;
var num_batches = 0;
while (pos < train_indices.Length)
{
var pos_end = Math.Min(pos + batch_size, train_indices.Length);
var minibatch_images = Util.get_tensors(features.Shape, train_features, train_indices, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(labels.Shape, train_labels, train_indices, pos, pos_end, computeDevice);
var feed_dictionary = new feed_t()
{
{ features, minibatch_images }, { labels, minibatch_labels }
};
trainer.TrainMinibatch(feed_dictionary, false, computeDevice);
epoch_training_error += trainer.PreviousMinibatchEvaluationAverage();
num_batches++;
pos = pos_end;
}
epoch_training_error /= num_batches;
training_accuracy.Add(1.0 - epoch_training_error);
// evaluation phase
var epoch_validation_error = 0.0;
num_batches = 0;
pos = 0;
while (pos < validation_features.Length)
{
var pos_end = Math.Min(pos + batch_size, validation_features.Length);
var minibatch_images = Util.get_tensors(features.Shape, validation_features, pos, pos_end, computeDevice);
var minibatch_labels = Util.get_tensors(labels.Shape, validation_labels, pos, pos_end, computeDevice);
var feed_dictionary = new test_feed_t()
{
{ features, minibatch_images }, { labels, minibatch_labels }
};
epoch_validation_error += evaluator.TestMinibatch(feed_dictionary, computeDevice);
pos = pos_end;
num_batches++;
}
epoch_validation_error /= num_batches;
validation_accuracy.Add(1.0 - epoch_validation_error);
Console.WriteLine($"Epoch {epoch + 1:D2}/{max_epochs}, training_accuracy={1.0-epoch_training_error:F3}, validation accuracy:{1-epoch_validation_error:F3}");
if (epoch_training_error < 0.001)
{
break;
}
}
return(new List <List <double> >()
{
training_accuracy, validation_accuracy
});
}
void train()
{
if (System.IO.File.Exists(model_filename))
{
System.Console.WriteLine("Model exists: " + model_filename);
return;
}
create_model();
load_data();
var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters());
var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.001), new CNTK.TrainingParameterScheduleDouble(0.9));
var trainer = CNTK.CNTKLib.CreateTrainer(model, loss, loss, new CNTK.LearnerVector()
{
learner
});
var evaluator = CNTK.CNTKLib.CreateEvaluator(loss);
var batch_size = 16;
var epochs = 10;
Console.WriteLine("Training on: " + computeDevice.AsString());
for (int current_epoch = 0; current_epoch < epochs; current_epoch++)
{
var epoch_start_time = DateTime.Now;
var epoch_training_loss = 0.0;
{
var train_indices = Util.shuffled_indices(x_train.Length);
var pos = 0;
while (pos < train_indices.Length)
{
var pos_end = Math.Min(pos + batch_size, train_indices.Length);
var x_minibatch = Util.get_tensors(input_img.Shape, x_train, train_indices, pos, pos_end, computeDevice);
var feed_dictionary = new Dictionary <CNTK.Variable, CNTK.Value> {
{ input_img, x_minibatch }
};
trainer.TrainMinibatch(feed_dictionary, false, computeDevice);
epoch_training_loss += (pos_end - pos) * trainer.PreviousMinibatchLossAverage();
pos = pos_end;
x_minibatch.Erase(); x_minibatch.Dispose(); x_minibatch = null;
}
epoch_training_loss /= x_train.Length;
}
var epoch_validation_loss = 0.0;
{
var pos = 0;
while (pos < x_test.Length)
{
var pos_end = Math.Min(pos + batch_size, x_test.Length);
var x_minibatch = Util.get_tensors(input_img.Shape, x_test, pos, pos_end, computeDevice);
var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr()
{
{ input_img, x_minibatch }
};
epoch_validation_loss += (pos_end - pos) * evaluator.TestMinibatch(feed_dictionary, computeDevice);
pos = pos_end;
x_minibatch.Erase(); x_minibatch.Dispose(); x_minibatch = null;
}
epoch_validation_loss /= x_test.Length;
}
var elapsedTime = DateTime.Now.Subtract(epoch_start_time);
Console.Write($"Epoch {current_epoch + 1:D2}/{epochs}, Elapsed time: {elapsedTime.TotalSeconds:F3} seconds. ");
Console.WriteLine($"Training loss: {epoch_training_loss:F3}. Validation Loss: {epoch_validation_loss:F3}.");
}
model.Save(model_filename);
}
