public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
{
// Evaluates.
var testData = new TextLoader(_testDataPath).CreateFrom <SentimentData>();
// BinaryClassificationEvaluator computes the quality metrics for the PredictionModel
// using the specified data set.
var evaluator = new BinaryClassificationEvaluator();
// BinaryClassificationMetrics contains the overall metrics computed by binary classification evaluators.
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
// The Accuracy metric gets the accuracy of a classifier, which is the proportion
// of correct predictions in the test set.
// The Auc metric gets the area under the ROC curve.
// The area under the ROC curve is equal to the probability that the classifier ranks
// a randomly chosen positive instance higher than a randomly chosen negative one
// (assuming 'positive' ranks higher than 'negative').
// The F1Score metric gets the classifier's F1 score.
// The F1 score is the harmonic mean of precision and recall:
// 2 * precision * recall / (precision + recall).
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
private static void Evaluate(PredictionModel <TitanicData, TitanicPrediction> model)
{
// To evaluate how good the model predicts values, the model is ran against new set
// of data (test data) that was not involved in training.
var testData = new TextLoader(TestDataPath).CreateFrom <TitanicData>(useHeader: true, separator: ',');
// BinaryClassificationEvaluator performs evaluation for Binary Classification type of ML problems.
var evaluator = new BinaryClassificationEvaluator();
Console.WriteLine("=============== Evaluating model ===============");
var metrics = evaluator.Evaluate(model, testData);
// BinaryClassificationMetrics contains the overall metrics computed by binary classification evaluators
// The Accuracy metric gets the accuracy of a classifier which is the proportion
//of correct predictions in the test set.
// The Auc metric gets the area under the ROC curve.
// The area under the ROC curve is equal to the probability that the classifier ranks
// a randomly chosen positive instance higher than a randomly chosen negative one
// (assuming 'positive' ranks higher than 'negative').
// The F1Score metric gets the classifier's F1 score.
// The F1 score is the harmonic mean of precision and recall:
// 2 * precision * recall / (precision + recall).
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
Console.WriteLine("=============== End evaluating ===============");
Console.WriteLine();
}
public static void CalcularModelo()
{
var pipeline = new LearningPipeline();
pipeline.Add(new TextLoader <SentimentData>(_dataPath, useHeader: false, separator: "tab"));
pipeline.Add(new TextFeaturizer("Features", "SentimentText"));
pipeline.Add(new FastTreeBinaryClassifier()
{
NumLeaves = 5, NumTrees = 5, MinDocumentsInLeafs = 2
});
PredictionModel <SentimentData, SentimentPrediction> model = pipeline.Train <SentimentData, SentimentPrediction>();
var testData = new TextLoader <SentimentData>(_testDataPath, useHeader: false, separator: "tab");
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
Console.WriteLine();
IEnumerable <SentimentData> sentiments = new[] {
new SentimentData
{
SentimentText = "Contoso's 11 is a wonderful experience",
Sentiment = 0
},
new SentimentData
{
SentimentText = "The acting in this movie is very bad",
Sentiment = 0
},
new SentimentData
{
SentimentText = "Joe versus the Volcano Coffee Company is a great film.",
Sentiment = 0
}
};
IEnumerable <SentimentPrediction> predictions = model.Predict(sentiments);
Console.WriteLine();
Console.WriteLine("Sentiment Predictions");
Console.WriteLine("---------------------");
var sentimentsAndPredictions = sentiments.Zip(predictions, (sentiment, prediction) => (sentiment, prediction));
foreach (var item in sentimentsAndPredictions)
{
Console.WriteLine($"Sentiment: {item.sentiment.SentimentText} | Prediction: {(item.prediction.Sentiment ? "Positive" : "Negative")}");
}
Console.WriteLine();
}
public LotteryPredictionResult PredictionOneToFile(string webRootPath, string noSite, string noType, TrainingData data, string lotteryCode)
{
var pipeline = new LearningPipeline();
string dataPath = webRootPath + $"/TrainingGround/{noSite}{noType}.txt";
pipeline.Add(new TextLoader(dataPath).CreateFrom <TrainingData>(separator: ','));
pipeline.Add(new Dictionarizer("Label"));
pipeline.Add(new ColumnConcatenator("Features", TrainingData.GetColumns()));
pipeline.Add(new LogisticRegressionBinaryClassifier());
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
_logger.LogInformation("Start PredictionOne :" + lotteryCode + "—" + noSite + noType);
var model = pipeline.Train <TrainingData, LotteryPrediction>();
var testData = new TextLoader(dataPath).CreateFrom <TrainingData>(separator: ',');
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
TrainingData newPoint = data;
LotteryPrediction prediction = model.Predict(newPoint);
string result = prediction.PredictedLabels;
_logger.LogInformation("End PredictionOne :" + lotteryCode + "—" + noSite + noType);
return(new LotteryPredictionResult()
{
PredictionType = noType,
PredictionSite = noSite,
PredictionResult = result,
LotteryCode = lotteryCode
});
}
public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
{
var testData = new List <SentimentData>()
{
new SentimentData {
Sentiment = 6f,
SentimentText = "such good thing"
},
new SentimentData {
Sentiment = -9.3f,
SentimentText = "f*****g article"
}
};
var collection = CollectionDataSource.Create(testData);
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, collection);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
public static void GetMyPrediction()
{
var pipeline = new LearningPipeline();
string dataPath = AppDomain.CurrentDomain.BaseDirectory + "/datamodel/myMLData.txt";
pipeline.Add(new TextLoader(dataPath).CreateFrom <myData>(separator: ' '));
pipeline.Add(new Dictionarizer("Label"));
pipeline.Add(new ColumnConcatenator("Features", "XCoord", "YCoord", "ZCoord"));
pipeline.Add(new LogisticRegressionBinaryClassifier());
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
Console.WriteLine("\nStarting training\n");
var model = pipeline.Train <myData, myPrediction>();
var testData = new TextLoader(dataPath).CreateFrom <myData>(separator: ' ');
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
double acc = metrics.Accuracy * 100;
Console.WriteLine("Model accuracy = " + acc.ToString("F2") + "%");
myData newPoint = new myData()
{
x = 9,
y = 8,
z = 10
};
myPrediction prediction = model.Predict(newPoint);
string result = prediction.PredictedLabels;
Console.WriteLine("Prediction = " + result);
Console.WriteLine("\nEnd ML.NET demo");
Console.ReadLine();
}
/// <summary>
/// Evaluates the trained model for quality assurance against a second data set.
///
/// Loads the test dataset.
/// Creates the binary evaluator.
/// Evaluates the model and create metrics.
///
/// Displays the metrics.
/// </summary>
/// <param name="model"></param>
internal static void Evaluate(
PredictionModel <ClassificationData, ClassPrediction> model,
InputData input)
{
// loads the new test dataset with the same schema.
// You can evaluate the model using this dataset as a quality check.
//var testData = new TextLoader(_testDataPath).CreateFrom<SentimentData>();
var testData = new TextLoader(input.TestData).CreateFrom <ClassificationData>();
// Computes the quality metrics for the PredictionModel using the specified dataset.
var evaluator = new BinaryClassificationEvaluator();
// The BinaryClassificationMetrics contains the overall metrics computed by binary
// classification evaluators. To display these to determine the quality of the model,
// you need to get the metrics first.
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
// Displaying the metrics for model validation
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($" Auc: {metrics.Auc:P2}");
Console.WriteLine($" F1Score: {metrics.F1Score:P2}");
}
public void Evaluation()
{
var dataPath = GetDataPath(SentimentDataPath);
var testDataPath = GetDataPath(SentimentDataPath);
var pipeline = new Legacy.LearningPipeline();
var loader = new TextLoader(dataPath).CreateFrom <SentimentData>();
loader.Arguments.HasHeader = true;
pipeline.Add(loader);
pipeline.Add(MakeSentimentTextTransform());
pipeline.Add(new FastTreeBinaryClassifier()
{
NumLeaves = 5, NumTrees = 5, MinDocumentsInLeafs = 2
});
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
var model = pipeline.Train <SentimentData, SentimentPrediction>();
var testLearningPipelineItem = new TextLoader(testDataPath).CreateFrom <SentimentData>();
testLearningPipelineItem.Arguments.HasHeader = true;
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testLearningPipelineItem);
}
public static void Execute()
{
Console.WriteLine("Executing Diabetes Experiment");
Console.WriteLine("Creating new model");
var pipeline = new LearningPipeline();
pipeline.Add(new TextLoader <DiabetesData>(dataPath, separator: ","));
var features = new string[] { "BMI", "Age", "Pregnancies", "PlasmaGlucoseConcentration", "TricepsSkinFoldThickness" };
pipeline.Add(new ColumnConcatenator("Features", features));
var algorithm = new BinaryLogisticRegressor();
pipeline.Add(algorithm);
model = pipeline.Train <DiabetesData, DiabetesPrediction>();
var testData = new TextLoader <DiabetesData>(testDataPath, separator: ",");
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
var score = metrics.Accuracy + metrics.Auc + metrics.F1Score;
double previousHighScore = 0;
if (File.Exists(modelStatsPath))
{
var previousModelData = File.ReadAllLines(modelStatsPath);
previousHighScore = double.Parse(previousModelData[0]);
}
if (score > previousHighScore)
{
File.WriteAllText(modelStatsPath, score.ToString() + Environment.NewLine);
File.AppendAllLines(modelStatsPath, new List <string>
{
$"Accuracy: {metrics.Accuracy:P2}",
$"Auc: {metrics.Auc:P2}",
$"F1Score: {metrics.F1Score:P2}"
});
File.AppendAllText(modelStatsPath, "Features:" + Environment.NewLine);
File.AppendAllLines(modelStatsPath, features);
File.AppendAllText(modelStatsPath, "Algorithm: " + algorithm.GetType().Name);
model.WriteAsync(modelPath);
Console.WriteLine("New model is better");
}
else
{
Console.WriteLine("Old model is better");
}
Console.ReadLine();
}
private void TestModel()
{
var evaluator = new BinaryClassificationEvaluator();
var testData = new TextLoader <ManifestDataTraining>(testPath, useHeader: true, separator: ";");
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"Accuracy = {metrics.Accuracy}");
}
public BinaryClassificationMetrics Test(string testDataPath, PredictionModel <Data, Prediction> model)
{
var testData = new TextLoader(testDataPath).CreateFrom <Data>();
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
return(metrics);
}
public BinaryClassificationMetrics Evaluate()
{
var testData = new TextLoader(TestDataPath).CreateFrom <SentimentData>();
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(_model, testData);
return(metrics);
}
public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
{
var testData = new TextLoader(_testDataPath).CreateFrom <SentimentData>();
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
public void TrainAndPredictSentimentModelTest()
{
var pipeline = PreparePipeline();
var model = pipeline.Train <SentimentData, SentimentPrediction>();
var testData = PrepareTextLoaderTestData();
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
ValidateExamples(model);
ValidateBinaryMetrics(metrics);
}
public static void Evalua(PredictionModel <DatosSentimiento, PredictSentimiento> modelo)
{
var datosPrueba = (new TextLoader <DatosSentimiento>(_rutaDatosEntrenamiento, useHeader: false, separator: "tab"));
var evaluador = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metricas = evaluador.Evaluate(modelo, datosPrueba);
Console.WriteLine();
Console.WriteLine("Evaluación de métricas de calidad del modelo de Predicción");
Console.WriteLine("--------------------------------");
Console.WriteLine($"Precisión: {metricas.Accuracy:P2}");
Console.WriteLine($"AUC: {metricas.Auc:P2}");
}
TrainAndGetMetrics(ILearningPipelineLoader dataTrain, ILearningPipelineLoader dataTest, ILearningPipelineItem trainer)
{
var pipeline = new LearningPipeline();
pipeline.Add(dataTrain);
pipeline.Add(trainer);
var model = pipeline.Train <MLNetData, MLNetPredict>();
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, dataTest);
return(metrics.Accuracy, metrics.Auc, metrics.F1Score, model);
}
public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
{
// Carga el conjunto de datos de prueba.
var testData = new TextLoader(_testDataPath).CreateFrom <SentimentData>();
// Crea el evaluador binario.
var evaluator = new BinaryClassificationEvaluator();
// Evalúa el modelo y crea métricas.
evaluator.Evaluate(model, testData);
// Muestra las métricas.
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model)
{
var testData = new TextLoader <SentimentData>(_testDataPath, useHeader: false, separator: "tab");
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
public static void Evaluate(PredictionModel <DiabetesData, DiabetesPrediction> model)
{
var testData = new TextLoader(_testDataPath).CreateFrom <DiabetesData>(separator: ',');
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
}
static void Main(string[] args)
{
var pipeline = new LearningPipeline();
var loader = new TextLoader(dataPath).CreateFrom <SentimentData>(useHeader: true, '\t');
pipeline.Add(loader);
pipeline.Add(new TextFeaturizer("Features", "SentimentText")
{
StopWordsRemover = new PredefinedStopWordsRemover(),
KeepPunctuations = false,
TextCase = TextNormalizerTransformCaseNormalizationMode.Lower,
VectorNormalizer = TextTransformTextNormKind.L2
});
pipeline.Add(new StochasticDualCoordinateAscentBinaryClassifier()
{
NumThreads = 8, Shuffle = true, NormalizeFeatures = NormalizeOption.Yes
});
PredictionModel <SentimentData, SentimentPrediction> model = pipeline.Train <SentimentData, SentimentPrediction>();
IEnumerable <SentimentData> sentiments = new[]
{
new SentimentData
{
SentimentText = "I hated the movie."
},
new SentimentData
{
SentimentText = "The movie was entertaining the whole time, i really enjoyed it."
}
};
IEnumerable <SentimentPrediction> predictions = model.Predict(sentiments);
foreach (var item in predictions)
{
Console.WriteLine($"Prediction: {(item.Sentiment ? "Positive" : "Negative")}");
}
var evulatorTrained = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metricsTrained = evulatorTrained.Evaluate(model, loader);
Console.WriteLine("ACCURACY OF MODEL ON TRAINED DATA: " + metricsTrained.Accuracy);
model.WriteAsync(trainedModelPath);
Console.Read();
}
private static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model, string name)
{
var testData = new TextLoader(TestDataPath).CreateFrom <SentimentData>();
var evaluator = new BinaryClassificationEvaluator();
Console.WriteLine("=============== Evaluating model {0} ===============", name);
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
Console.WriteLine("=============== End evaluating ===============");
Console.WriteLine();
}
static void Main(string[] args)
{
var trainDataPath = Path.Combine(Environment.CurrentDirectory, "Data", "requestClassifier-trainData.tsv");
var testDataPath = Path.Combine(Environment.CurrentDirectory, "Data", "requestClassifier-testData.tsv");
var modelPath = Path.Combine(Environment.CurrentDirectory, "Data", "Model.zip");
Console.WriteLine("Welcome! Let's predict which department to forward each requests to. As of now we have 2 departments: Administration and Registration");
Console.WriteLine("Initialize pipeline by loading training data, editing metadata, and selecting ML algorithm");
var pipeline = new LearningPipeline()
{
new TextLoader(trainDataPath).CreateFrom <UserRequest>(useHeader: true),
new TextFeaturizer("Features", "Question"),
new FastTreeBinaryClassifier()
{
NumLeaves = 5, NumTrees = 5, MinDocumentsInLeafs = 2
}
};
Console.WriteLine("Let's train our model with all the specs in our learning pipeline and we'll write it to model to disk");
var model = pipeline.Train <UserRequest, DepartmentPrepiction>();
model.WriteAsync(modelPath).Wait();
Console.WriteLine("Let's test our model with test data to see exactly how it performs");
var testData = new TextLoader(testDataPath).CreateFrom <UserRequest>(useHeader: true);
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
Console.WriteLine("Now let's try to use our model to with live data");
do
{
Console.WriteLine("Ask a question now");
string question = Console.ReadLine();
var prediction = model.Predict(new UserRequest {
Question = question
});
model.TryGetScoreLabelNames(out string[] data);
Console.WriteLine($"Predicted Department: {prediction}");
Console.WriteLine("Press <ENTER> to continue");
}while (Console.ReadKey().Key == ConsoleKey.Enter);
}
public static void Evalua(PredictionModel <DatosSentimiento, PrediccSentimiento> modelo)
{
var datosPrueba = new TextLoader <DatosSentimiento>(_rutaDatosPrueba, useHeader: false, separator: "tab");
var evaluador = new BinaryClassificationEvaluator(); //Obtener Metricas de evaluacion
BinaryClassificationMetrics metricas = evaluador.Evaluate(modelo, datosPrueba); //modelo:es el modelo de prediccion entrenado que vamos a evaluar
Console.WriteLine();
Console.WriteLine("Evaluación de métricas de calidad del Modelo de Predicción");
Console.WriteLine("---------------------------------");
Console.WriteLine($"Precisión: {metricas.Accuracy:P2}"); //La presicion indica que tan acertado ha sido el algoritmo durante la prediccion
Console.WriteLine($"AUC: {metricas.Auc:P2}"); //Medida del rendimiento para´problemas de clasificacion binaria (1.0 correcto)
Console.WriteLine($"Log-loss: {metricas.LogLoss:P2}");
Console.WriteLine($"F1SCore: {metricas.F1Score:P2}");
//AUC=78%-> de cada 100 elementos se han clasificado 78 correctamente
}
public void Evaluate(PredictionModel <NrlResult, ClusterPrediction> model,
IEnumerable <NrlResult> nrlResults)
{
var testData = CollectionDataSource.Create(nrlResults);
var evaluator = new BinaryClassificationEvaluator();
Console.WriteLine("=============== Evaluating model ===============");
var metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
Console.WriteLine("=============== End evaluating ===============");
Console.WriteLine();
}
static void Main(string[] args)
{
//1. Build an ML.NET pipeline for training a sentiment analysis model
Console.WriteLine("Training a model for Sentiment Analysis using ML.NET");
var pipeline = new LearningPipeline();
// 1a. Load the training data using a TextLoader.
pipeline.Add(new TextLoader(@"..\..\..\Data\wikipedia-detox-250-line-data.tsv").CreateFrom <SentimentData>(useHeader: true));
// 1b. Featurize the text into a numeric vector that can be used by the machine learning algorithm.
pipeline.Add(new TextFeaturizer("Features", "SentimentText"));
// 1c. Add AveragedPerceptron (a linear learner) to the pipeline.
pipeline.Add(new AveragedPerceptronBinaryClassifier()
{
NumIterations = 10
});
// 1d. Get a model by training the pipeline that was built.
PredictionModel <SentimentData, SentimentPrediction> model =
pipeline.Train <SentimentData, SentimentPrediction>();
// 2. Evaluate the model to see how well it performs on different data (output the percent of examples classified correctly).
Console.WriteLine("Training of model is complete \nTesting the model with test data");
var testData = new TextLoader(@"..\..\..\Data\wikipedia-detox-250-line-test.tsv").CreateFrom <SentimentData>(useHeader: true);
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine($"Accuracy of trained model for test data is: {metrics.Accuracy:P2}");
// 3. Save the model to file so it can be used in another app.
model.WriteAsync("sentiment_model.zip");
// 4. Use the model for a single prediction.
SentimentData testInput = new SentimentData {
SentimentText = "ML.NET is fun, more samples at https://github.com/dotnet/machinelearning-samples"
};
var sentiment = (model.Predict(testInput).Sentiment == true) ? "Positive" : "Negative";
/* This template uses a minimal dataset to build a sentiment analysis model which leads to relatively low accuracy.
* In order to build a sentiment analysis model with higher accuracy please follow the walkthrough at https://aka.ms/mlnetsentimentanalysis*/
Console.WriteLine("Predicted sentiment for \"" + testInput.SentimentText + "\" is:" + sentiment);
Console.ReadKey();
}
public static void Evaluate(PredictionModel <SentimentData, SentimentPrediction> model, IMongoDatabase db)
{
// Evaluates.
// <Snippet13>
var collection = db.GetCollection <SentimentData>("review_test");
var documents = collection.Find <SentimentData>(new BsonDocument()).ToEnumerable();
var testData = CollectionDataSource.Create(documents);
// </Snippet13>
// BinaryClassificationEvaluator computes the quality metrics for the PredictionModel
// using the specified data set.
// <Snippet14>
var evaluator = new BinaryClassificationEvaluator();
// </Snippet14>
// BinaryClassificationMetrics contains the overall metrics computed by binary classification evaluators.
// <Snippet15>
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
// </Snippet15>
// The Accuracy metric gets the accuracy of a classifier, which is the proportion
// of correct predictions in the test set.
// The Auc metric gets the area under the ROC curve.
// The area under the ROC curve is equal to the probability that the classifier ranks
// a randomly chosen positive instance higher than a randomly chosen negative one
// (assuming 'positive' ranks higher than 'negative').
// The F1Score metric gets the classifier's F1 score.
// The F1 score is the harmonic mean of precision and recall:
// 2 * precision * recall / (precision + recall).
// <Snippet16>
Console.WriteLine();
Console.WriteLine("PredictionModel quality metrics evaluation");
Console.WriteLine("------------------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.Auc:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
// </Snippet16>
}
static void Main(string[] args)
{
string trainSetPath = "train_data.csv";
string testSetPath = "test_data.csv";
var pipeline = new LearningPipeline();
pipeline.Add(new TextLoader <Passenger>(trainSetPath, useHeader: true, separator: ","));
pipeline.Add(new ColumnDropper()
{
Column = new string[] { "Cabin", "Ticket" }
});
pipeline.Add(new MissingValueSubstitutor(new string[] { "Age" })
{
ReplacementKind = NAReplaceTransformReplacementKind.Mean
});
pipeline.Add(new CategoricalOneHotVectorizer("Sex", "Embarked"));
pipeline.Add(new ColumnConcatenator(
"Features", "Age", "Pclass", "SibSp", "Parch", "Sex", "Embarked"));
pipeline.Add(new FastTreeBinaryClassifier());
var model = pipeline.Train <Passenger, PredictedData>();
var testLoader = new TextLoader <Passenger>(testSetPath, useHeader: true, separator: ",");
var evaluator = new BinaryClassificationEvaluator();
var metrics = evaluator.Evaluate(model, testLoader);
Console.WriteLine($"Accuracy: {metrics.Accuracy} F1 Score: {metrics.F1Score}");
Console.WriteLine($"True Positive: {metrics.ConfusionMatrix[0, 0]} False Positive: {metrics.ConfusionMatrix[0, 1]}");
Console.WriteLine($"False Negative: {metrics.ConfusionMatrix[1, 0]} True Negative: {metrics.ConfusionMatrix[1, 1]}");
}
public void TrainAndPredictSentimentModelTest()
{
string dataPath = GetDataPath(SentimentDataPath);
var pipeline = new LearningPipeline();
pipeline.Add(new Data.TextLoader(dataPath)
{
Arguments = new TextLoaderArguments
{
Separator = new[] { '\t' },
HasHeader = true,
Column = new[]
{
new TextLoaderColumn()
{
Name = "Label",
Source = new [] { new TextLoaderRange(0) },
Type = Runtime.Data.DataKind.Num
},
new TextLoaderColumn()
{
Name = "SentimentText",
Source = new [] { new TextLoaderRange(1) },
Type = Runtime.Data.DataKind.Text
}
}
}
});
pipeline.Add(new TextFeaturizer("Features", "SentimentText")
{
KeepDiacritics = false,
KeepPunctuations = false,
TextCase = TextNormalizerTransformCaseNormalizationMode.Lower,
OutputTokens = true,
StopWordsRemover = new PredefinedStopWordsRemover(),
VectorNormalizer = TextTransformTextNormKind.L2,
CharFeatureExtractor = new NGramNgramExtractor()
{
NgramLength = 3, AllLengths = false
},
WordFeatureExtractor = new NGramNgramExtractor()
{
NgramLength = 2, AllLengths = true
}
});
pipeline.Add(new FastTreeBinaryClassifier()
{
NumLeaves = 5, NumTrees = 5, MinDocumentsInLeafs = 2
});
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
PredictionModel <SentimentData, SentimentPrediction> model = pipeline.Train <SentimentData, SentimentPrediction>();
IEnumerable <SentimentData> sentiments = new[]
{
new SentimentData
{
SentimentText = "Please refrain from adding nonsense to Wikipedia."
},
new SentimentData
{
SentimentText = "He is a CHEATER, and the article should say that."
}
};
IEnumerable <SentimentPrediction> predictions = model.Predict(sentiments);
Assert.Equal(2, predictions.Count());
Assert.True(predictions.ElementAt(0).Sentiment.IsFalse);
Assert.True(predictions.ElementAt(1).Sentiment.IsTrue);
string testDataPath = GetDataPath(SentimentTestPath);
var testData = new Data.TextLoader(testDataPath)
{
Arguments = new TextLoaderArguments
{
Separator = new[] { '\t' },
HasHeader = true,
Column = new[]
{
new TextLoaderColumn()
{
Name = "Label",
Source = new [] { new TextLoaderRange(0) },
Type = Runtime.Data.DataKind.Num
},
new TextLoaderColumn()
{
Name = "SentimentText",
Source = new [] { new TextLoaderRange(1) },
Type = Runtime.Data.DataKind.Text
}
}
}
};
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Assert.Equal(.5556, metrics.Accuracy, 4);
Assert.Equal(.8, metrics.Auc, 1);
Assert.Equal(.87, metrics.Auprc, 2);
Assert.Equal(1, metrics.Entropy, 3);
Assert.Equal(.6923, metrics.F1Score, 4);
Assert.Equal(.969, metrics.LogLoss, 3);
Assert.Equal(3.083, metrics.LogLossReduction, 3);
Assert.Equal(1, metrics.NegativePrecision, 3);
Assert.Equal(.111, metrics.NegativeRecall, 3);
Assert.Equal(.529, metrics.PositivePrecision, 3);
Assert.Equal(1, metrics.PositiveRecall);
ConfusionMatrix matrix = metrics.ConfusionMatrix;
Assert.Equal(2, matrix.Order);
Assert.Equal(2, matrix.ClassNames.Count);
Assert.Equal("positive", matrix.ClassNames[0]);
Assert.Equal("negative", matrix.ClassNames[1]);
Assert.Equal(9, matrix[0, 0]);
Assert.Equal(9, matrix["positive", "positive"]);
Assert.Equal(0, matrix[0, 1]);
Assert.Equal(0, matrix["positive", "negative"]);
Assert.Equal(8, matrix[1, 0]);
Assert.Equal(8, matrix["negative", "positive"]);
Assert.Equal(1, matrix[1, 1]);
Assert.Equal(1, matrix["negative", "negative"]);
}
static void Main(string[] args)
{
try
{
//var text = System.IO.File.ReadAllLines(_dataPath);
var pipeline = new LearningPipeline();
pipeline.Add(new TextLoader <FlightData>(_dataPath, useHeader: true, separator: ","));
pipeline.Add(new ColumnCopier(("DepDel15", "Label")));
pipeline.Add(new MissingValuesRowDropper()
{
Column = new string[]
{
"Year",
"Month",
"DayofMonth",
"DayOfWeek",
"Carrier",
"OriginAirportID",
"DestAirportID",
"CRSDepTime",
"DepDelay",
"DepDel15",
"CRSArrTime",
"ArrDelay",
"ArrDel15",
"Cancelled"
}
});
pipeline.Add(new CategoricalOneHotVectorizer(
"Month",
"DayofMonth",
"DayOfWeek",
// "Carrier",
"OriginAirportID",
"DestAirportID"//,
//"CRSDepTime"
));
pipeline.Add(new ColumnConcatenator("Features",
"Month",
"DayofMonth",
"DayOfWeek",
//"Carrier",
"OriginAirportID",
"DestAirportID" //,
//"CRSDepTime"
//"DepDelay"
));
pipeline.Add(new FastTreeBinaryClassifier()
{
UnbalancedSets = true
});
var model = pipeline.Train <FlightData, FlightDelayPredictor>();
model.WriteAsync(_modelPath);
var testData = new TextLoader <FlightData>(_dataPath, useHeader: true, separator: ",");
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
//model.Predict(new FlightData() { })
model.WriteAsync(_modelPath);
var testRows = System.IO.File.ReadAllLines(_dataPath);
foreach (var testRow in testRows)
{
var objects = testRow.Split(",");
var obj = new FlightData()
{
Month = objects[1],
DayofMonth = objects[2],
DayOfWeek = objects[3],
OriginAirportID = objects[5],
DestAirportID = objects[6],
CRSDepTime = objects[7]
};
var prediction = model.Predict(obj);
if (prediction.DepDel15 != 0)
{
Console.WriteLine("Predicted DepartureDelay is: {0}", prediction.DepDel15);
}
}
//var prediction = model.Predict(new FlightData()
//{ //2013,6,30,7,AA,13930,10721,1855,64.00,1.00,2215,45.00,1.00,0.00
// Year = "2013",
// Month = "6",
// DayofMonth = "30",
// DayOfWeek = "7",
// Carrier = "AA",
// OriginAirportID = "13930",
// DestAirportID = "10721",
// CRSDepTime = "1855",
// DepDelay =64.00f,
// //DepDel15 = 1.00f,
// CRSArrTime = 2215,
// ArrDel15 = 1.00f,
// Cancelled = 0.00f
//}
//);
//Console.WriteLine("Predicted DepartureDelay is: {0}", prediction.DepDel15);
}
catch (Exception ex)
{
}
}
public static void Run()
{
// Define pipeline
var pipeline = new LearningPipeline();
pipeline.Add(new TextLoader("1_BinaryClassification/problem1_train.csv").CreateFrom <BeerOrWineData>(useHeader: true, separator: ','));
pipeline.Add(new TextFeaturizer("Features", "FullName"));
pipeline.Add(new Dictionarizer(("Type", "Label")));
pipeline.Add(new StochasticDualCoordinateAscentBinaryClassifier()
{
});
//pipeline.Add(new FastTreeBinaryClassifier() { NumLeaves = 25, NumTrees = 25, MinDocumentsInLeafs = 2 }); // up to 91%
//pipeline.Add(new FastForestBinaryClassifier() { NumLeaves = 25, NumTrees = 25, MinDocumentsInLeafs = 2 }); // 86%
//pipeline.Add(new StochasticDualCoordinateAscentBinaryClassifier() { }); // 95%
//pipeline.Add(new StochasticGradientDescentBinaryClassifier { }); // 92%
pipeline.Add(new PredictedLabelColumnOriginalValueConverter()
{
PredictedLabelColumn = "PredictedLabel"
});
// Train model
var stopWatch = new Stopwatch();
stopWatch.Start();
var model = pipeline.Train <BeerOrWineData, BeerOrWinePrediction>();
stopWatch.Stop();
Console.WriteLine($"Trained the model in: {stopWatch.ElapsedMilliseconds / 1000} seconds.");
// Evaluate model
var testData = new TextLoader("1_BinaryClassification/problem1_validate.csv").CreateFrom <BeerOrWineData>(useHeader: true, separator: ',');
var evaluator = new BinaryClassificationEvaluator();
BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData);
Console.WriteLine(metrics.Accuracy.ToString("P"));
// show matrix
// Use model
IEnumerable <BeerOrWineData> drinks = new[]
{
new BeerOrWineData {
FullName = "Castle Stout"
},
new BeerOrWineData {
FullName = "Folkes Röda IPA"
},
new BeerOrWineData {
FullName = "Fryken Havre Ale"
},
new BeerOrWineData {
FullName = "Barolo Gramolere"
},
new BeerOrWineData {
FullName = "Château de Lavison"
},
new BeerOrWineData {
FullName = "Korlat Cabernet Sauvignon"
}
};
var predictions = model.Predict(drinks).ToList();
}