private static void Main(string[] args)
{
//Create the MLContext to share across components for deterministic results
MLContext mlContext = new MLContext(seed: 1); //Seed set to any number so you have a deterministic environment
// STEP 1: Common data loading configuration
IDataView fullData = mlContext.Data.ReadFromTextFile(path: DataPath,
columns: new[]
{
new TextLoader.Column(DefaultColumnNames.Label, DataKind.R4, 0),
new TextLoader.Column(nameof(IrisData.SepalLength), DataKind.R4, 1),
new TextLoader.Column(nameof(IrisData.SepalWidth), DataKind.R4, 2),
new TextLoader.Column(nameof(IrisData.PetalLength), DataKind.R4, 3),
new TextLoader.Column(nameof(IrisData.PetalWidth), DataKind.R4, 4),
},
hasHeader: true,
separatorChar: '\t');
//Split dataset in two parts: TrainingDataset (80%) and TestDataset (20%)
(IDataView trainingDataView, IDataView testingDataView) = mlContext.Clustering.TrainTestSplit(fullData, testFraction: 0.2);
//STEP 2: Process data transformations in pipeline
var dataProcessPipeline = mlContext.Transforms.Concatenate(DefaultColumnNames.Features, nameof(IrisData.SepalLength), nameof(IrisData.SepalWidth), nameof(IrisData.PetalLength), nameof(IrisData.PetalWidth));
// (Optional) Peek data in training DataView after applying the ProcessPipeline's transformations
Common.ConsoleHelper.PeekDataViewInConsole(mlContext, trainingDataView, dataProcessPipeline, 10);
Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, DefaultColumnNames.Features, trainingDataView, dataProcessPipeline, 10);
// STEP 3: Create and train the model
var trainer = mlContext.Clustering.Trainers.KMeans(featureColumn: DefaultColumnNames.Features, clustersCount: 3);
var trainingPipeline = dataProcessPipeline.Append(trainer);
var trainedModel = trainingPipeline.Fit(trainingDataView);
// STEP4: Evaluate accuracy of the model
IDataView predictions = trainedModel.Transform(testingDataView);
var metrics = mlContext.Clustering.Evaluate(predictions, score: DefaultColumnNames.Score, features: DefaultColumnNames.Features);
ConsoleHelper.PrintClusteringMetrics(trainer.ToString(), metrics);
// STEP5: Save/persist the model as a .ZIP file
using (var fs = new FileStream(ModelPath, FileMode.Create, FileAccess.Write, FileShare.Write))
mlContext.Model.Save(trainedModel, fs);
Console.WriteLine("=============== End of training process ===============");
Console.WriteLine("=============== Predict a cluster for a single case (Single Iris data sample) ===============");
// Test with one sample text
var sampleIrisData = new IrisData()
{
SepalLength = 3.3f,
SepalWidth = 1.6f,
PetalLength = 0.2f,
PetalWidth = 5.1f,
};
using (var stream = new FileStream(ModelPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
ITransformer model = mlContext.Model.Load(stream);
// Create prediction engine related to the loaded trained model
var predEngine = model.CreatePredictionEngine <IrisData, IrisPrediction>(mlContext);
//Score
var resultprediction = predEngine.Predict(sampleIrisData);
Console.WriteLine($"Cluster assigned for setosa flowers:" + resultprediction.SelectedClusterId);
}
Console.WriteLine("=============== End of process, hit any key to finish ===============");
Console.ReadKey();
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2);
// Register Types in IoC container for DI
//MLContext created as singleton for the whole ASP.NET Core app
services.AddSingleton <MLContext>((ctx) =>
{
//Seed set to any number so you have a deterministic environment
return(new MLContext(seed: 1));
});
//ML Model (ITransformed) created as singleton for the whole ASP.NET Core app. Loads from .zip file here.
services.AddSingleton <ITransformer,
TransformerChain <ITransformer> > ((ctx) =>
{
MLContext mlContext = ctx.GetRequiredService <MLContext>();
string modelFilePathName = Configuration["MLModel:MLModelFilePath"];
ITransformer mlModel;
using (var fileStream = File.OpenRead(modelFilePathName))
mlModel = mlContext.Model.Load(fileStream);
return((TransformerChain <ITransformer>)mlModel);
});
// PredictionEngine created as Transient since it is not thread safe
// This injected PredictionEngine is ONLY used on the MLModelEngineSimple implementation
// This injected PredictionEngine is NOT used when using the Object Pooling or ThreadStatic approaches
services.AddTransient <PredictionEngine <SampleObservation, SamplePrediction> >((ctx) =>
{
MLContext mlContext = ctx.GetRequiredService <MLContext>();
ITransformer mlmodel = ctx.GetRequiredService <ITransformer>();
var predEngine = mlmodel.CreatePredictionEngine <SampleObservation, SamplePrediction>(mlContext);
return(predEngine);
});
// OPTION A:
// Using MLModelEngine ObjPooling implementation
//
services.AddSingleton <IMLModelEngine <SampleObservation, SamplePrediction>,
MLModelEngineObjPooling <SampleObservation, SamplePrediction> >();
// OPTION B:
// Using MLModelEngine ThreadStatic implementation
//
//services.AddSingleton<IMLModelEngine<SampleObservation, SamplePrediction>,
// MLModelEngineThreadStatic<SampleObservation, SamplePrediction>>();
// OPTION C:
// Using MLModelEngine Simple implementation (Create Prediction Engine for every call)
//
//services.AddSingleton<IMLModelEngine<SampleObservation, SamplePrediction>,
// MLModelEngineSimple<SampleObservation, SamplePrediction>>();
// Using 'Factory code' when registering the engine.
// Not needed in current implementation
//
//services.AddSingleton<IMLModelEngine<SampleObservation, SamplePrediction>,
// MLModelEngineObjPooling<SampleObservation, SamplePrediction>>((ctx) =>
//{
// MLContext mlContext = ctx.GetRequiredService<MLContext>();
// string modelFilePathName = Configuration["MLModelFilePath"];
// return new MLModelEngineObjPooling<SampleObservation, SamplePrediction>(mlContext,
// modelFilePathName);
//});
}
public PredictionBaseModel Predict(ITransformer trainedModel, BaseModel model)
{
var engine = trainedModel.CreatePredictionEngine <BaseModel, PredictionBaseModel>(mlContext);
return(engine.Predict(model));
}
public Task <TrainedModelPrediction> UseModelWithSingleItem(ITransformer model, Flute.Shared.Models.TrainedModel sample)
{
PredictionEngine <TrainedModel, TrainedModelPrediction> predictionFunction = model.CreatePredictionEngine <TrainedModel, TrainedModelPrediction>(mlContext);
var resultprediction = predictionFunction.Predict(new TrainedModel()
{
Input = sample.Input, Label = Convert.ToBoolean(sample.Label)
});
return(Task.FromResult(resultprediction));
}
public StudentPredictionModel Predict(ITransformer trainedModel, StudentTrainingModel model)
{
var engine = trainedModel.CreatePredictionEngine <StudentTrainingModel, StudentPredictionModel>(mlContext);
return(engine.Predict(model));
}
public PredictionEngine <TData, TPrediction> Create()
{
var predictionEngine = _model.CreatePredictionEngine <TData, TPrediction>(_mlContext);
return(predictionEngine);
}
private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
{
//user model with single data time
PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine <SentimentData, SentimentPrediction>(mlContext);
SentimentData sampleStatement = new SentimentData
{
SentimentText = "This was a very bad steak"
};
var resultprediction = predictionFunction.Predict(sampleStatement);
Console.WriteLine();
Console.WriteLine("=== Prediction test of model with a single sample and test dataset ===");
Console.WriteLine();
Console.WriteLine($"Sentiment: {sampleStatement.SentimentText} | Prediction: {(Convert.ToBoolean(resultprediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultprediction.Probability} ");
Console.WriteLine("=== End of predictions ===");
Console.WriteLine();
}
//static TextLoader _textLoader; not used
static void Main(string[] args)
{
// STEP 1: Common data loading configuration
MLContext mlContext = new MLContext(seed: 0);
IDataView baseTrainingDataView = mlContext.Data.LoadFromTextFile <TaxiTrip>(_trainDataPath, hasHeader: true, separatorChar: ','); // ReadFromTextFile is deprecated
IDataView testDataView = mlContext.Data.LoadFromTextFile <TaxiTrip>(_testDataPath, hasHeader: true, separatorChar: ',');
//Sample code of removing extreme data like "outliers" for FareAmounts higher than $150 and lower than $1 which can be error-data
var cnt = baseTrainingDataView.GetColumn <float>(mlContext, nameof(TaxiTrip.FareAmount)).Count();
IDataView trainingDataView = mlContext.Data.FilterRowsByColumn(baseTrainingDataView, nameof(TaxiTrip.FareAmount), lowerBound: 1, upperBound: 150);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessPipeline = mlContext.Transforms.CopyColumns(outputColumnName: DefaultColumnNames.Label, inputColumnName: nameof(TaxiTrip.FareAmount))
.Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "VendorIdEncoded", inputColumnName: nameof(TaxiTrip.VendorId)))
.Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "RateCodeEncoded", inputColumnName: nameof(TaxiTrip.RateCode)))
.Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "PaymentTypeEncoded", inputColumnName: nameof(TaxiTrip.PaymentType)))
.Append(mlContext.Transforms.Normalize(outputColumnName: nameof(TaxiTrip.PassengerCount), mode: NormalizerMode.MeanVariance))
.Append(mlContext.Transforms.Normalize(outputColumnName: nameof(TaxiTrip.TripTime), mode: NormalizerMode.MeanVariance))
.Append(mlContext.Transforms.Normalize(outputColumnName: nameof(TaxiTrip.TripDistance), mode: NormalizerMode.MeanVariance))
.Append(mlContext.Transforms.Concatenate(DefaultColumnNames.Features, "VendorIdEncoded", "RateCodeEncoded", "PaymentTypeEncoded", nameof(TaxiTrip.PassengerCount)
, nameof(TaxiTrip.TripTime), nameof(TaxiTrip.TripDistance)));
// STEP 3: Set the training algorithm, then create and config the modelBuilder - Selected Trainer (SDCA Regression algorithm)
var trainer = mlContext.Regression.Trainers.StochasticDualCoordinateAscent(labelColumnName: DefaultColumnNames.Label, featureColumnName: DefaultColumnNames.Features);
var trainingPipeline = dataProcessPipeline.Append(trainer);
//STEP 4: Train the model fitting to the DataSet
//The pipeline is trained on the dataset that has been loaded and transformed.
ITransformer trainedModel = trainingPipeline.Fit(trainingDataView); // changed var trainedModel to ITransformer trainedModel
// STEP 5: Evaluate the model and show accuracy stats
// create a copy of the testDataView in IDataView format since IDataView is immutable?
IDataView predictions = trainedModel.Transform(testDataView);
// assign label column (original regression value) and score column (predictive regression values) to columns in data i.e. predictions
// apply metrics e.g. root mean squared error on data
var metrics = mlContext.Regression.Evaluate(predictions, label: DefaultColumnNames.Label, score: DefaultColumnNames.Score);
// print the results of the metrics
Common.ConsoleHelper.PrintRegressionMetrics(trainer.ToString(), metrics);
// STEP 6: Save/persist the trained model to a .ZIP file
using (var fs = File.Create(_modelPath))
trainedModel.SaveTo(mlContext, fs);
Console.WriteLine("The model is saved to {0}", _modelPath);
// STEP 7: Prediction
//Sample:
//vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount
//VTS,1,1,1140,3.75,CRD,15.5
var taxiTripSample = new TaxiTrip()
{
VendorId = "VTS",
RateCode = "1",
PassengerCount = 1,
TripTime = 1140,
TripDistance = 3.75f,
PaymentType = "CRD",
FareAmount = 0 // To predict. Actual/Observed = 15.5
};
using (var stream = new FileStream(_modelPath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
trainedModel = mlContext.Model.Load(stream);
}
// Create prediction engine related to the loaded trained model
var predEngine = trainedModel.CreatePredictionEngine <TaxiTrip, TaxiTripFarePrediction>(mlContext);
//Score
var resultprediction = predEngine.Predict(taxiTripSample);
Console.WriteLine($"**********************************************************************");
Console.WriteLine($"Predicted fare: {resultprediction.FareAmount:0.####}, actual fare: 15.5");
Console.WriteLine($"**********************************************************************");
}
/// <summary>
/// The main entry point of the program.
/// </summary>
/// <param name="args">The command line arguments.</param>
static void Main(string[] args)
{
// create a machine learning context
var context = new MLContext();
// load the dataset in memory
Console.WriteLine("Loading data...");
var data = context.Data.LoadFromTextFile <ProductInfo>(
dataPath,
hasHeader: true,
separatorChar: '\t');
// split the data into 80% training and 20% testing partitions
var partitions = context.BinaryClassification.TrainTestSplit(data, testFraction: 0.2);
// prepare matrix factorization options
var options = new MatrixFactorizationTrainer.Options()
{
MatrixColumnIndexColumnName = "ProductIDEncoded",
MatrixRowIndexColumnName = "CombinedProductIDEncoded",
LabelColumnName = "CombinedProductID",
LossFunction = MatrixFactorizationTrainer.LossFunctionType.SquareLossOneClass,
Alpha = 0.01,
Lambda = 0.025,
};
// set up a training pipeline
// step 1: map ProductID and CombinedProductID to keys
var pipeline = context.Transforms.Conversion.MapValueToKey(
inputColumnName: "ProductID",
outputColumnName: "ProductIDEncoded")
.Append(context.Transforms.Conversion.MapValueToKey(
inputColumnName: "CombinedProductID",
outputColumnName: "CombinedProductIDEncoded"))
// step 2: find recommendations using matrix factorization
.Append(context.Recommendation().Trainers.MatrixFactorization(options));
// train the model
Console.WriteLine("Training the model...");
ITransformer model = pipeline.Fit(partitions.TrainSet);
Console.WriteLine();
// evaluate the model performance
Console.WriteLine("Evaluating the model...");
var predictions = model.Transform(partitions.TestSet);
var metrics = context.Regression.Evaluate(predictions, label: "CombinedProductID", score: DefaultColumnNames.Score);
Console.WriteLine($" RMSE: {metrics.Rms:#.##}");
Console.WriteLine($" L1: {metrics.L1:#.##}");
Console.WriteLine($" L2: {metrics.L2:#.##}");
Console.WriteLine();
// check how well products 3 and 63 go together
Console.WriteLine("Predicting if two products combine...");
var engine = model.CreatePredictionEngine <ProductInfo, ProductPrediction>(context);
var prediction = engine.Predict(
new ProductInfo()
{
ProductID = 3,
CombinedProductID = 63
});
Console.WriteLine($" Score of products 3 and 63 combined: {prediction.Score}");
Console.WriteLine();
// find the top 5 combined products for product 6
Console.WriteLine("Calculating the top 5 products for product 3...");
var top5 = (from m in Enumerable.Range(1, 262111)
let p = engine.Predict(
new ProductInfo()
{
ProductID = 3,
CombinedProductID = (uint)m
})
orderby p.Score descending
select(ProductID: m, Score: p.Score)).Take(5);
foreach (var t in top5)
{
Console.WriteLine($" Score:{t.Score}\tProduct: {t.ProductID}");
}
Console.ReadLine();
}
public PredictionEngine <SentimentIssue, SentimentPrediction> CreatePredictionEngine()
{
return(_trainedModel.CreatePredictionEngine <SentimentIssue, SentimentPrediction>(_context));
}
public Predictor(MLContext context, ITransformer containsPeopleModel, ITransformer numberOfPeopleModel, ITransformer distanceModel)
{
ContainsPeoplePredictor = containsPeopleModel.CreatePredictionEngine <Data, PredictedContainsPeople>(context);
NumberOfPeoplePredictor = numberOfPeopleModel.CreatePredictionEngine <Data, PredictedNumberOfPeople>(context);
DistancePredictor = distanceModel.CreatePredictionEngine <Data, PredictedDistance>(context);
}
private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
{
PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine <SentimentData, SentimentPrediction>(mlContext);
// The Predict() function makes a prediction on a single row of data.
var res = predictionFunction.Predict(new SentimentData()
{
SentimentText = "This was a very bad steak"
});
Console.WriteLine("=============== Prediction Test of model with a single sample and test dataset ===============");
Console.WriteLine($"Prediction {res.Prediction}");
Console.WriteLine("=============== End of Predictions ===============");
Console.WriteLine();
}
private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
{
//PredictionEngine<SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine<SentimentData, SentimentPrediction>(mlContext);
PredictionEngine <TrabajoPlanificadoPropuestaData, TrabajoPlanificadoPrediction> predictionFunction = model.CreatePredictionEngine <TrabajoPlanificadoPropuestaData, TrabajoPlanificadoPrediction>(mlContext);
TrabajoPlanificadoPropuestaData sampleStatement = new TrabajoPlanificadoPropuestaData();
var resultprediction = predictionFunction.Predict(sampleStatement);
Console.WriteLine();
Console.WriteLine("=============== Prediction Test of model with a single sample and test dataset ===============");
Console.WriteLine();
//Console.WriteLine($"Sentiment: {sampleStatement.SentimentText} | Prediction: {(Convert.ToBoolean(resultprediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultprediction.Probability} ");
Console.WriteLine("=============== End of Predictions ===============");
Console.WriteLine();
}
public static SentimentPrediction RunModel(this MLContext mlContext, ITransformer model, string sampleText)
{
PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine <SentimentData, SentimentPrediction>(mlContext);
return(predictionFunction.Predict(new SentimentData {
SentimentText = sampleText
}));
}