private void TestNeuralNetwork(string testCsvPath, int nb_classes, Dictionary <string, int> dictionaryLikeIMDB, int max_news_len) { NDarray x_test = null; NDarray y_test = null; var testCSV = Frame.ReadCsv(testCsvPath, false, separators: ";"); var testYFloat = testCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <float>("Column1")); }).ValuesAll.ToList(); var testXString = testCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <string>("Column2")); }).ValuesAll.ToList(); var testXStringArray = testXString.ToArray(); y_test = np.array(testYFloat.ToArray()); y_test = Util.ToCategorical(y_test, nb_classes); string[][] tokens_test = testXStringArray.Tokenize(); int[][] bow_test = FrequencyDictionary.Transform(tokens_test, dictionaryLikeIMDB); //double[][] bow_test = codebook.Transform(tokens_test); var list_test = new List <NDarray>(); foreach (var item in bow_test) { //var newItem = item.Take(100).ToArray(); //var ndarray = np.array(newItem); var ndarray = np.array(item); list_test.Add(ndarray); } var sequences_test = np.array(list_test); x_test = SequenceUtil.PadSequences(sequences_test, maxlen: max_news_len); //Load model and weight var loaded_model = Sequential.ModelFromJson(File.ReadAllText("model.json")); loaded_model.LoadWeight("best_model_gru.h5"); loaded_model.Compile(optimizer: "adam", loss: "categorical_crossentropy", metrics: new string[] { "accuracy" }); loaded_model.Summary(); var scores = loaded_model.Evaluate(x_test, y_test, verbose: 0); Console.WriteLine("Test loss:" + scores[0] * 100); Console.WriteLine("Test accuracy:" + scores[1] * 100); }
private (History, Sequential, Dictionary <string, int>) LearnNeuralNetwork(string trainCsvPath, int num_words, int max_news_len, int nb_classes) { NDarray x_train = null; NDarray y_train = null; var trainCSV = Frame.ReadCsv(trainCsvPath, false, separators: ";"); var trainYFloat = trainCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <float>("Column1")); }).ValuesAll.ToList(); var trainXString = trainCSV.Rows.Select(kvp => { return(kvp.Value.GetAs <string>("Column2")); }).ValuesAll.ToList(); var trainXStringArray = trainXString.ToArray(); //x_train = np.array(new float[,] { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } }); y_train = np.array(trainYFloat.ToArray()); y_train = Util.ToCategorical(y_train, nb_classes); string[][] tokens = trainXStringArray.Tokenize(); var dictionaryLikeIMDB = FrequencyDictionary.Learn(tokens); var bow = FrequencyDictionary.Transform(tokens, dictionaryLikeIMDB); // Create a new TF-IDF with options: /*var codebook = new Accord.MachineLearning.TFIDF() * { * Tf = TermFrequency.Log, * Idf = InverseDocumentFrequency.Default * }; * * codebook.Learn(tokens); * * double[][] bow = codebook.Transform(tokens);*/ var list = new List <NDarray>(); foreach (var item in bow) { //var newItem = item.Take(max_news_len).ToArray(); //var ndarray = np.array(newItem); var ndarray = np.array(item); list.Add(ndarray); } var sequences = np.array(list); //x_train = SequenceUtil.PadSequences(sequences, maxlen: max_news_len, dtype: "double"); x_train = SequenceUtil.PadSequences(sequences, maxlen: max_news_len); var model = new Sequential(); model.Add(new Embedding(num_words, 32, null, null, null, null, false, max_news_len)); model.Add(new GRU(138));//16 model.Add(new Dense(12, activation: "softmax")); model.Compile(optimizer: "adam", loss: "categorical_crossentropy", metrics: new string[] { "accuracy" }); model.Summary(); var model_gru_save_path = "best_model_gru.h5"; var checkpoint_callback_gru = new ModelCheckpoint( model_gru_save_path, "val_accuracy", 1, true ); var callbacks = new List <Callback>() { checkpoint_callback_gru }; float validation_split = (float)0.1; var history_gru = model.Fit(x_train, y_train, batch_size: 128, epochs: 10, validation_split: validation_split, callbacks: callbacks.ToArray()); //Save model and weights string json = model.ToJson(); File.WriteAllText("model.json", json); return(history_gru, model, dictionaryLikeIMDB); }