void evaluate_network() { var batch_size = 128; var pos = 0; var accuracy = 0.0; var num_batches = 0; var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function); while (pos < test_images.Length) { var pos_end = Math.Min(pos + batch_size, test_images.Length); var minibatch_images = Util.get_tensors(image_tensor.Shape, test_images, pos, pos_end, computeDevice); var minibatch_labels = Util.get_tensors(label_tensor.Shape, test_labels, pos, pos_end, computeDevice); var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr() { { image_tensor, minibatch_images }, { label_tensor, minibatch_labels } }; var minibatch_accuracy = evaluator.TestMinibatch(feed_dictionary, computeDevice); accuracy += minibatch_accuracy; pos = pos_end; num_batches++; } accuracy /= num_batches; Console.WriteLine(string.Format("Accuracy:{0:F3}", accuracy)); }
void train_network() { int epochs = 5; int batch_size = 64; for (int current_epoch = 0; current_epoch < epochs; current_epoch++) { // training phase var train_indices = Util.shuffled_indices(60000); var pos = 0; while (pos < train_indices.Length) { var pos_end = Math.Min(pos + batch_size, train_indices.Length); var minibatch_images = Util.get_tensors(image_tensor.Shape, train_images, train_indices, pos, pos_end, computeDevice); var minibatch_labels = Util.get_tensors(label_tensor.Shape, train_labels, train_indices, pos, pos_end, computeDevice); var feed_dictionary = new feed_t() { { image_tensor, minibatch_images }, { label_tensor, minibatch_labels } }; trainer.TrainMinibatch(feed_dictionary, false, computeDevice); pos = pos_end; } // evaluation phase var accuracy = 0.0; var num_batches = 0; pos = 0; while (pos < test_images.Length) { var pos_end = Math.Min(pos + batch_size, test_images.Length); var minibatch_images = Util.get_tensors(image_tensor.Shape, test_images, pos, pos_end, computeDevice); var minibatch_labels = Util.get_tensors(label_tensor.Shape, test_labels, pos, pos_end, computeDevice); var feed_dictionary = new test_feed_t() { { image_tensor, minibatch_images }, { label_tensor, minibatch_labels } }; var minibatch_accuracy = evaluator.TestMinibatch(feed_dictionary, computeDevice); accuracy += minibatch_accuracy; pos = pos_end; num_batches++; } accuracy /= num_batches; Console.WriteLine(string.Format("Epoch {0}/{1}, Test accuracy:{2:F3}", current_epoch + 1, epochs, 1.0 - accuracy)); } }
double evaluation_phase(int[] validation_indices) { var pos = 0; var num_batches = 0; var epoch_evaluation_accuracy = 0.0; while (pos < validation_indices.Length) { var pos_end = Math.Min(pos + batch_size, validation_indices.Length); var minibatch_x = Util.get_tensors(x_tensor.Shape, x_train, validation_indices, pos, pos_end, computeDevice); var minibatch_y = Util.get_tensors(y_tensor.Shape, y_train, validation_indices, pos, pos_end, computeDevice); var feed_dictionary = new test_feed_t() { { x_tensor, minibatch_x }, { y_tensor, minibatch_y } }; var minibatch_accuracy = evaluator.TestMinibatch(feed_dictionary, computeDevice); epoch_evaluation_accuracy += minibatch_accuracy; num_batches++; pos = pos_end; } epoch_evaluation_accuracy /= num_batches; return(epoch_evaluation_accuracy); }
List <List <double> > train_with_extracted_features() { float[][] train_features = null, validation_features = null, test_features = null; float[][] train_labels = null, validation_labels = null, test_labels = null; compute_features_and_labels(ref train_features, ref validation_features, ref test_features, ref train_labels, ref validation_labels, ref test_labels); var labels = CNTK.Variable.InputVariable(new int[] { 2, }, CNTK.DataType.Float, "labels_var"); var features = CNTK.Variable.InputVariable(new int[] { extracted_feature_length, }, CNTK.DataType.Float, "features_var"); var model = CNTK.CNTKLib.ReLU(Util.Dense(features, 256, computeDevice)); model = CNTK.CNTKLib.Dropout(model, 0.5); model = Util.Dense(model, 2, computeDevice); var loss_function = CNTK.CNTKLib.CrossEntropyWithSoftmax(model.Output, labels); var accuracy_function = CNTK.CNTKLib.ClassificationError(model.Output, labels); var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters()); var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.0001, 1), new CNTK.TrainingParameterScheduleDouble(0.99, 1)); var trainer = CNTK.Trainer.CreateTrainer(model, loss_function, accuracy_function, new CNTK.Learner[] { learner }); var evaluator = CNTK.CNTKLib.CreateEvaluator(accuracy_function); var training_accuracy = new List <double>(); var validation_accuracy = new List <double>(); for (int epoch = 0; epoch < max_epochs; epoch++) { // training phase var epoch_training_error = 0.0; var train_indices = Util.shuffled_indices(train_features.Length); var pos = 0; var num_batches = 0; while (pos < train_indices.Length) { var pos_end = Math.Min(pos + batch_size, train_indices.Length); var minibatch_images = Util.get_tensors(features.Shape, train_features, train_indices, pos, pos_end, computeDevice); var minibatch_labels = Util.get_tensors(labels.Shape, train_labels, train_indices, pos, pos_end, computeDevice); var feed_dictionary = new feed_t() { { features, minibatch_images }, { labels, minibatch_labels } }; trainer.TrainMinibatch(feed_dictionary, false, computeDevice); epoch_training_error += trainer.PreviousMinibatchEvaluationAverage(); num_batches++; pos = pos_end; } epoch_training_error /= num_batches; training_accuracy.Add(1.0 - epoch_training_error); // evaluation phase var epoch_validation_error = 0.0; num_batches = 0; pos = 0; while (pos < validation_features.Length) { var pos_end = Math.Min(pos + batch_size, validation_features.Length); var minibatch_images = Util.get_tensors(features.Shape, validation_features, pos, pos_end, computeDevice); var minibatch_labels = Util.get_tensors(labels.Shape, validation_labels, pos, pos_end, computeDevice); var feed_dictionary = new test_feed_t() { { features, minibatch_images }, { labels, minibatch_labels } }; epoch_validation_error += evaluator.TestMinibatch(feed_dictionary, computeDevice); pos = pos_end; num_batches++; } epoch_validation_error /= num_batches; validation_accuracy.Add(1.0 - epoch_validation_error); Console.WriteLine($"Epoch {epoch + 1:D2}/{max_epochs}, training_accuracy={1.0-epoch_training_error:F3}, validation accuracy:{1-epoch_validation_error:F3}"); if (epoch_training_error < 0.001) { break; } } return(new List <List <double> >() { training_accuracy, validation_accuracy }); }
List <List <double> > fit_generator(bool sequence_mode, CNTK.Variable x, CNTK.Variable y, CNTK.Function model, CNTK.Trainer trainer, CNTK.Evaluator evaluator, GeneratorsInfo gi, int epochs, int steps_per_epoch, CNTK.DeviceDescriptor computeDevice) { var history = new List <List <double> >() { new List <double>(), new List <double>() }; var train_enumerator = gi.train_gen.GetEnumerator(); var val_enumerator = gi.val_gen.GetEnumerator(); var x_minibatch_dims = new List <int>(x.Shape.Dimensions); if (sequence_mode == false) { x_minibatch_dims.Add(gi.batch_size); } for (int current_epoch = 0; current_epoch < epochs; current_epoch++) { var epoch_start_time = DateTime.Now; var epoch_training_error = 0.0; { var num_total_samples = 0; for (int s = 0; s < steps_per_epoch; s++) { train_enumerator.MoveNext(); var st = train_enumerator.Current; var x_minibatch = create_x_minibatch(sequence_mode, x, gi, st, computeDevice); var y_minibatch = CNTK.Value.CreateBatch(y.Shape, st.targets, computeDevice); var feed_dictionary = new Dictionary <CNTK.Variable, CNTK.Value> { { x, x_minibatch }, { y, y_minibatch } }; bool isSweepEndInArguments = (s == (steps_per_epoch - 1)); trainer.TrainMinibatch(feed_dictionary, isSweepEndInArguments, computeDevice); var minibatch_metric = trainer.PreviousMinibatchEvaluationAverage(); epoch_training_error += minibatch_metric * st.targets.Length; num_total_samples += st.targets.Length; x_minibatch.Erase(); y_minibatch.Erase(); } epoch_training_error /= num_total_samples; } history[0].Add(epoch_training_error); var epoch_validation_error = 0.0; { var num_total_samples = 0; for (int s = 0; s < gi.val_steps; s++) { val_enumerator.MoveNext(); var st = val_enumerator.Current; var x_minibatch = create_x_minibatch(sequence_mode, x, gi, st, computeDevice); var y_minibatch = CNTK.Value.CreateBatch(y.Shape, st.targets, computeDevice); var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr() { { x, x_minibatch }, { y, y_minibatch } }; var minibatch_metric = evaluator.TestMinibatch(feed_dictionary, computeDevice); epoch_validation_error += minibatch_metric * st.targets.Length; num_total_samples += st.targets.Length; x_minibatch.Erase(); y_minibatch.Erase(); } epoch_validation_error /= num_total_samples; } history[1].Add(epoch_validation_error); var elapsedTime = DateTime.Now.Subtract(epoch_start_time); Console.WriteLine($"Epoch {current_epoch + 1:D2}/{epochs}, Elapsed time: {elapsedTime.TotalSeconds:F3} seconds. " + $"Training Error: {epoch_training_error:F3}. Validation Error: {epoch_validation_error:F3}."); } return(history); }
void train() { if (System.IO.File.Exists(model_filename)) { System.Console.WriteLine("Model exists: " + model_filename); return; } create_model(); load_data(); var pv = new CNTK.ParameterVector((System.Collections.ICollection)model.Parameters()); var learner = CNTK.CNTKLib.AdamLearner(pv, new CNTK.TrainingParameterScheduleDouble(0.001), new CNTK.TrainingParameterScheduleDouble(0.9)); var trainer = CNTK.CNTKLib.CreateTrainer(model, loss, loss, new CNTK.LearnerVector() { learner }); var evaluator = CNTK.CNTKLib.CreateEvaluator(loss); var batch_size = 16; var epochs = 10; Console.WriteLine("Training on: " + computeDevice.AsString()); for (int current_epoch = 0; current_epoch < epochs; current_epoch++) { var epoch_start_time = DateTime.Now; var epoch_training_loss = 0.0; { var train_indices = Util.shuffled_indices(x_train.Length); var pos = 0; while (pos < train_indices.Length) { var pos_end = Math.Min(pos + batch_size, train_indices.Length); var x_minibatch = Util.get_tensors(input_img.Shape, x_train, train_indices, pos, pos_end, computeDevice); var feed_dictionary = new Dictionary <CNTK.Variable, CNTK.Value> { { input_img, x_minibatch } }; trainer.TrainMinibatch(feed_dictionary, false, computeDevice); epoch_training_loss += (pos_end - pos) * trainer.PreviousMinibatchLossAverage(); pos = pos_end; x_minibatch.Erase(); x_minibatch.Dispose(); x_minibatch = null; } epoch_training_loss /= x_train.Length; } var epoch_validation_loss = 0.0; { var pos = 0; while (pos < x_test.Length) { var pos_end = Math.Min(pos + batch_size, x_test.Length); var x_minibatch = Util.get_tensors(input_img.Shape, x_test, pos, pos_end, computeDevice); var feed_dictionary = new CNTK.UnorderedMapVariableValuePtr() { { input_img, x_minibatch } }; epoch_validation_loss += (pos_end - pos) * evaluator.TestMinibatch(feed_dictionary, computeDevice); pos = pos_end; x_minibatch.Erase(); x_minibatch.Dispose(); x_minibatch = null; } epoch_validation_loss /= x_test.Length; } var elapsedTime = DateTime.Now.Subtract(epoch_start_time); Console.Write($"Epoch {current_epoch + 1:D2}/{epochs}, Elapsed time: {elapsedTime.TotalSeconds:F3} seconds. "); Console.WriteLine($"Training loss: {epoch_training_loss:F3}. Validation Loss: {epoch_validation_loss:F3}."); } model.Save(model_filename); }