private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
{
// STEP 1: Common data loading configuration
var textLoader = SentimentAnalysysTextLoaderFactory.CreateTextLoader(mlContext);
var trainingDataView = textLoader.Read(TrainDataPath);
var testDataView = textLoader.Read(TestDataPath);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessPipeline = mlContext.Transforms.Text.FeaturizeText("Text", "Features");
// STEP 3: Set the training algorithm, then create and config the modelBuilder
var modelBuilder = new Common.ModelBuilder <SentimentIssue, SentimentPrediction>(mlContext, dataProcessPipeline);
var trainer = mlContext.BinaryClassification.Trainers.FastTree(label: "Label", features: "Features");
modelBuilder.AddTrainer(trainer);
// STEP 4: Train the model fitting to the DataSet
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 5: Evaluate the model and show accuracy stats
Console.WriteLine("===== Evaluating Model's accuracy with Test data =====");
var metrics = modelBuilder.EvaluateBinaryClassificationModel(testDataView, "Label", "Score");
Common.ConsoleHelper.PrintBinaryClassificationMetrics(trainer.ToString(), metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
return(modelBuilder.TrainedModel);
}
private static void BuildTrainEvaluateAndSaveModel(MLContext mlContext)
{
// STEP 1: Common data loading configuration
var textLoader = IrisTextLoaderFactory.CreateTextLoader(mlContext);
var trainingDataView = textLoader.Read(TrainDataPath);
var testDataView = textLoader.Read(TestDataPath);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessPipeline = mlContext.Transforms.Concatenate("Features", "SepalLength",
"SepalWidth",
"PetalLength",
"PetalWidth");
// STEP 3: Set the training algorithm, then create and config the modelBuilder
var modelBuilder = new Common.ModelBuilder <IrisData, IrisPrediction>(mlContext, dataProcessPipeline);
// We apply our selected Trainer
var trainer = mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent(labelColumn: "Label", featureColumn: "Features");
modelBuilder.AddTrainer(trainer);
// STEP 4: Train the model fitting to the DataSet
//The pipeline is trained on the dataset that has been loaded and transformed.
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 5: Evaluate the model and show accuracy stats
Console.WriteLine("===== Evaluating Model's accuracy with Test data =====");
var metrics = modelBuilder.EvaluateMultiClassClassificationModel(testDataView, "Label");
Common.ConsoleHelper.PrintMultiClassClassificationMetrics(trainer.ToString(), metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
}
static void Main(string[] args)
{
//Create MLContext to be shared across the model creation workflow objects
//Set a random seed for repeatable/deterministic results across multiple trainings.
var mlContext = new MLContext();
// STEP 1: Common data loading configuration
DataLoader dataLoader = new DataLoader(mlContext);
var trainingDataView = dataLoader.GetDataView(TrainDataPath);
var testDataView = dataLoader.GetDataView(TestDataPath);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessor = new DataProcessor(mlContext);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (OPTIONAL) Peek data (such as 2 records) in training DataView after applying the ProcessPipeline's transformations into "Features"
Common.ConsoleHelper.PeekDataViewInConsole <SentimentIssue>(mlContext, trainingDataView, dataProcessPipeline, 2);
//Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 2);
// STEP 3: Set the training algorithm, then create and config the modelBuilder
var modelBuilder = new Common.ModelBuilder <SentimentIssue, SentimentPrediction>(mlContext, dataProcessPipeline);
var trainer = mlContext.BinaryClassification.Trainers.StochasticDualCoordinateAscent(label: "Label", features: "Features");
//Other way: var trainer = new LinearClassificationTrainer(mlContext, "Features", "Label");
modelBuilder.AddTrainer(trainer);
// STEP 4: Train the model fitting to the DataSet
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 5: Evaluate the model and show accuracy stats
Console.WriteLine("===== Evaluating Model's accuracy with Test data =====");
var metrics = modelBuilder.EvaluateBinaryClassificationModel(testDataView, "Label", "Score");
Common.ConsoleHelper.PrintBinaryClassificationMetrics("StochasticDualCoordinateAscent", metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
// (OPTIONAL) Try/test a single prediction by loding the model from the file, first.
SentimentIssue sampleStatement = new SentimentIssue {
Text = "This is a very rude movie"
};
var modelScorer = new Common.ModelScorer <SentimentIssue, SentimentPrediction>(mlContext);
modelScorer.LoadModelFromZipFile(ModelPath);
var resultprediction = modelScorer.PredictSingle(sampleStatement);
Console.WriteLine($"=============== Single Prediction ===============");
Console.WriteLine($"Text: {sampleStatement.Text} | Prediction: {(Convert.ToBoolean(resultprediction.Prediction) ? "Toxic" : "Nice")} sentiment | Probability: {resultprediction.Probability} ");
Console.WriteLine($"==================================================");
//
Common.ConsoleHelper.ConsoleWriteHeader("=============== End of training process, hit any key to finish ===============");
Console.ReadKey();
}
static void Main(string[] args)
{
var assetsPath = ModelHelpers.GetAssetsPath(@"..\..\..\assets");
var transactionsCsv = Path.Combine(assetsPath, "inputs", "transactions.csv");
var offersCsv = Path.Combine(assetsPath, "inputs", "offers.csv");
var pivotCsv = Path.Combine(assetsPath, "inputs", "pivot.csv");
var modelZip = Path.Combine(assetsPath, "outputs", "retailClustering.zip");
try
{
//DataHelpers.PreProcessAndSave(offersCsv, transactionsCsv, pivotCsv);
//var modelBuilder = new ModelBuilder(pivotCsv, modelZip, kValuesSvg);
//modelBuilder.BuildAndTrain();
//STEP 0: Special data pre-process in this sample creating the PivotTable csv file
DataHelpers.PreProcessAndSave(offersCsv, transactionsCsv, pivotCsv);
//Create the MLContext to share across components for deterministic results
MLContext mlContext = new MLContext(seed: 1); //Seed set to any number so you have a deterministic environment
//STEP 1: Common data loading
DataLoader dataLoader = new DataLoader(mlContext);
var pivotDataView = dataLoader.GetDataView(pivotCsv);
//STEP 2: Process data transformations in pipeline
var dataProcessor = new DataProcessor(mlContext, 2);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (Optional) Peek data in training DataView after applying the ProcessPipeline's transformations
Common.ConsoleHelper.PeekDataViewInConsole <PivotObservation>(mlContext, pivotDataView, dataProcessPipeline, 10);
Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", pivotDataView, dataProcessPipeline, 10);
// STEP 3: Create and train the model
var trainer = mlContext.Clustering.Trainers.KMeans("Features", clustersCount: 3);
var modelBuilder = new Common.ModelBuilder <PivotObservation, ClusteringPrediction>(mlContext, dataProcessPipeline);
modelBuilder.AddTrainer(trainer);
var trainedModel = modelBuilder.Train(pivotDataView);
// STEP4: Evaluate accuracy of the model
var metrics = modelBuilder.EvaluateClusteringModel(pivotDataView);
Common.ConsoleHelper.PrintClusteringMetrics("KMeans", metrics);
// STEP5: Save/persist the model as a .ZIP file
modelBuilder.SaveModelAsFile(modelZip);
} catch (Exception ex)
{
Common.ConsoleHelper.ConsoleWriteException(ex.Message);
}
Common.ConsoleHelper.ConsolePressAnyKey();
}
static void Main(string[] args)
{
var assetsPath = ModelHelpers.GetAssetsPath(@"..\..\..\assets");
var transactionsCsv = Path.Combine(assetsPath, "inputs", "transactions.csv");
var offersCsv = Path.Combine(assetsPath, "inputs", "offers.csv");
var pivotCsv = Path.Combine(assetsPath, "inputs", "pivot.csv");
var modelZip = Path.Combine(assetsPath, "outputs", "retailClustering.zip");
try
{
//STEP 0: Special data pre-process in this sample creating the PivotTable csv file
DataHelpers.PreProcessAndSave(offersCsv, transactionsCsv, pivotCsv);
//Create the MLContext to share across components for deterministic results
MLContext mlContext = new MLContext(seed: 1); //Seed set to any number so you have a deterministic environment
// STEP 1: Common data loading configuration
var textLoader = CustomerSegmentationTextLoaderFactory.CreateTextLoader(mlContext);
var pivotDataView = textLoader.Read(pivotCsv);
//STEP 2: Configure data transformations in pipeline
var dataProcessPipeline = new PrincipalComponentAnalysisEstimator(mlContext, "Features", "PCAFeatures", rank: 2)
.Append(new OneHotEncodingEstimator(mlContext, new[] { new OneHotEncodingEstimator.ColumnInfo("LastName",
"LastNameKey",
CategoricalTransform.OutputKind.Ind) }));
// (Optional) Peek data in training DataView after applying the ProcessPipeline's transformations
Common.ConsoleHelper.PeekDataViewInConsole <PivotObservation>(mlContext, pivotDataView, dataProcessPipeline, 10);
Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", pivotDataView, dataProcessPipeline, 10);
// STEP 3: Create and train the model
var trainer = mlContext.Clustering.Trainers.KMeans("Features", clustersCount: 3);
var modelBuilder = new Common.ModelBuilder <PivotObservation, ClusteringPrediction>(mlContext, dataProcessPipeline);
modelBuilder.AddTrainer(trainer);
var trainedModel = modelBuilder.Train(pivotDataView);
// STEP4: Evaluate accuracy of the model
var metrics = modelBuilder.EvaluateClusteringModel(pivotDataView);
Common.ConsoleHelper.PrintClusteringMetrics(trainer.ToString(), metrics);
// STEP5: Save/persist the model as a .ZIP file
modelBuilder.SaveModelAsFile(modelZip);
} catch (Exception ex)
{
Common.ConsoleHelper.ConsoleWriteException(ex.Message);
}
Common.ConsoleHelper.ConsolePressAnyKey();
}
private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
{
// STEP 1: Common data loading configuration
DataLoader dataLoader = new DataLoader(mlContext);
var trainingDataView = dataLoader.GetDataView(TrainDataPath);
var testDataView = dataLoader.GetDataView(TestDataPath);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessor = new DataProcessor(mlContext);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (OPTIONAL) Peek data (such as 5 records) in training DataView after applying the ProcessPipeline's transformations into "Features"
Common.ConsoleHelper.PeekDataViewInConsole <TaxiTrip>(mlContext, trainingDataView, dataProcessPipeline, 5);
Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 5);
// STEP 3: Set the training algorithm, then create and config the modelBuilder
var modelBuilder = new Common.ModelBuilder <TaxiTrip, TaxiTripFarePrediction>(mlContext, dataProcessPipeline);
// We apply our selected Trainer (SDCA Regression algorithm)
var trainer = mlContext.Regression.Trainers.StochasticDualCoordinateAscent(label: "Label", features: "Features");
modelBuilder.AddTrainer(trainer);
// STEP 4: Train the model fitting to the DataSet
//The pipeline is trained on the dataset that has been loaded and transformed.
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 5: Evaluate the model and show accuracy stats
Console.WriteLine("===== Evaluating Model's accuracy with Test data =====");
var metrics = modelBuilder.EvaluateRegressionModel(testDataView, "Label", "Score");
Common.ConsoleHelper.PrintRegressionMetrics("StochasticDualCoordinateAscent", metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
return(modelBuilder.TrainedModel);
}
public static void CreateTrainAndEvaluateModel(MLContext mlContext)
{
// STEP 1: Common data loading configuration
DataLoader dataLoader = new DataLoader(mlContext);
var trainingDataView = dataLoader.GetDataView(TrainDataPath);
var testDataView = dataLoader.GetDataView(TestDataPath);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessor = new DataProcessor(mlContext);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (OPTIONAL) Peek data (such as 2 records) in training DataView after applying the ProcessPipeline's transformations into "Features"
Common.ConsoleHelper.PeekDataViewInConsole <TitanicData>(mlContext, trainingDataView, dataProcessPipeline, 2);
Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 2);
// STEP 3: Set the training algorithm, then create and config the modelBuilder
// FastTreeBinaryClassifier is an algorithm that will be used to train the model.
// It has three hyperparameters for tuning decision tree performance.
//pipeline.Add(new FastTreeBinaryClassifier());// {NumLeaves = 5, NumTrees = 5, MinDocumentsInLeafs = 2});
var modelBuilder = new Common.ModelBuilder <TitanicData, TitanicPrediction>(mlContext, dataProcessPipeline);
var trainer = mlContext.BinaryClassification.Trainers.FastTree(label: "Label", features: "Features", numLeaves: 10, numTrees: 5, minDatapointsInLeafs: 10);
modelBuilder.AddTrainer(trainer);
// STEP 4: Train the model fitting to the DataSet
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 5: Evaluate the model and show accuracy stats
Console.WriteLine("===== Evaluating Model's accuracy with Test data =====");
var metrics = modelBuilder.EvaluateBinaryClassificationModel(testDataView, "Label", "Score");
Common.ConsoleHelper.PrintBinaryClassificationMetrics(trainer.ToString(), metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
}
private static ITransformer BuildTrainEvaluateAndSaveModel(MLContext mlContext)
{
// STEP 1: Common data loading configuration
DataLoader dataLoader = new DataLoader(mlContext);
var trainingDataView = dataLoader.GetDataView(TrainDataPath);
var testDataView = dataLoader.GetDataView(TestDataPath);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessor = new DataProcessor(mlContext);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (OPTIONAL) Peek data (such as 2 records) in training DataView after applying the ProcessPipeline's transformations into "Features"
Common.ConsoleHelper.PeekDataViewInConsole <SentimentIssue>(mlContext, trainingDataView, dataProcessPipeline, 2);
//Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 2);
// STEP 3: Set the training algorithm, then create and config the modelBuilder
var modelBuilder = new Common.ModelBuilder <SentimentIssue, SentimentPrediction>(mlContext, dataProcessPipeline);
var trainer = mlContext.BinaryClassification.Trainers.FastTree(label: "Label", features: "Features");
modelBuilder.AddTrainer(trainer);
// STEP 4: Train the model fitting to the DataSet
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 5: Evaluate the model and show accuracy stats
Console.WriteLine("===== Evaluating Model's accuracy with Test data =====");
var metrics = modelBuilder.EvaluateBinaryClassificationModel(testDataView, "Label", "Score");
Common.ConsoleHelper.PrintBinaryClassificationMetrics(trainer.ToString(), metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
return(modelBuilder.TrainedModel);
}
