private void TestNeuralNetwork(string testCsvPath, int nb_classes, Dictionary <string, int> dictionaryLikeIMDB, int max_news_len)
{
NDarray x_test = null;
NDarray y_test = null;
var testCSV = Frame.ReadCsv(testCsvPath, false, separators: ";");
var testYFloat = testCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <float>("Column1")); }).ValuesAll.ToList();
var testXString = testCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <string>("Column2")); }).ValuesAll.ToList();
var testXStringArray = testXString.ToArray();
y_test = np.array(testYFloat.ToArray());
y_test = Util.ToCategorical(y_test, nb_classes);
string[][] tokens_test = testXStringArray.Tokenize();
int[][] bow_test = FrequencyDictionary.Transform(tokens_test, dictionaryLikeIMDB);
//double[][] bow_test = codebook.Transform(tokens_test);
var list_test = new List <NDarray>();
foreach (var item in bow_test)
{
//var newItem = item.Take(100).ToArray();
//var ndarray = np.array(newItem);
var ndarray = np.array(item);
list_test.Add(ndarray);
}
var sequences_test = np.array(list_test);
x_test = SequenceUtil.PadSequences(sequences_test, maxlen: max_news_len);
//Load model and weight
var loaded_model = Sequential.ModelFromJson(File.ReadAllText("model.json"));
loaded_model.LoadWeight("best_model_gru.h5");
loaded_model.Compile(optimizer: "adam", loss: "categorical_crossentropy", metrics: new string[] { "accuracy" });
loaded_model.Summary();
var scores = loaded_model.Evaluate(x_test, y_test, verbose: 0);
Console.WriteLine("Test loss:" + scores[0] * 100);
Console.WriteLine("Test accuracy:" + scores[1] * 100);
}
private (History, Sequential, Dictionary <string, int>) LearnNeuralNetwork(string trainCsvPath, int num_words, int max_news_len, int nb_classes)
{
NDarray x_train = null;
NDarray y_train = null;
var trainCSV = Frame.ReadCsv(trainCsvPath, false, separators: ";");
var trainYFloat = trainCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <float>("Column1")); }).ValuesAll.ToList();
var trainXString = trainCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <string>("Column2")); }).ValuesAll.ToList();
var trainXStringArray = trainXString.ToArray();
//x_train = np.array(new float[,] { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } });
y_train = np.array(trainYFloat.ToArray());
y_train = Util.ToCategorical(y_train, nb_classes);
string[][] tokens = trainXStringArray.Tokenize();
var dictionaryLikeIMDB = FrequencyDictionary.Learn(tokens);
var bow = FrequencyDictionary.Transform(tokens, dictionaryLikeIMDB);
// Create a new TF-IDF with options:
/*var codebook = new Accord.MachineLearning.TFIDF()
* {
* Tf = TermFrequency.Log,
* Idf = InverseDocumentFrequency.Default
* };
*
* codebook.Learn(tokens);
*
* double[][] bow = codebook.Transform(tokens);*/
var list = new List <NDarray>();
foreach (var item in bow)
{
//var newItem = item.Take(max_news_len).ToArray();
//var ndarray = np.array(newItem);
var ndarray = np.array(item);
list.Add(ndarray);
}
var sequences = np.array(list);
//x_train = SequenceUtil.PadSequences(sequences, maxlen: max_news_len, dtype: "double");
x_train = SequenceUtil.PadSequences(sequences, maxlen: max_news_len);
var model = new Sequential();
model.Add(new Embedding(num_words, 32, null, null, null, null, false, max_news_len));
model.Add(new GRU(138));//16
model.Add(new Dense(12, activation: "softmax"));
model.Compile(optimizer: "adam", loss: "categorical_crossentropy", metrics: new string[] { "accuracy" });
model.Summary();
var model_gru_save_path = "best_model_gru.h5";
var checkpoint_callback_gru = new ModelCheckpoint(
model_gru_save_path,
"val_accuracy",
1,
true
);
var callbacks = new List <Callback>()
{
checkpoint_callback_gru
};
float validation_split = (float)0.1;
var history_gru = model.Fit(x_train,
y_train,
batch_size: 128,
epochs: 10,
validation_split: validation_split,
callbacks: callbacks.ToArray());
//Save model and weights
string json = model.ToJson();
File.WriteAllText("model.json", json);
return(history_gru, model, dictionaryLikeIMDB);
}
