public static void Main(string[] args)
{
var engine = new PredictionEngine<Statistics>();
engine.LoadDB(EngineDb);
var buffer = "";
var character = '\0';
var predictions = new Dictionary<char, float>();
do
{
var key = Console.ReadKey();
character = key.KeyChar;
if (character == ShellExit)
{
break;
}
if (engine.ValidCharacter(character) == false)
{
continue;
}
engine.CharacterTyped(character);
buffer += character;
Console.Clear();
Console.WriteLine(buffer);
predictions = engine.GetPredictions();
if (engine.IsUnknownWord())
{
Console.WriteLine("Unknown word!");
continue;
}
foreach (var prediction in predictions)
{
if (prediction.Value != 0)
{
Console.WriteLine("P(" + prediction.Key + ") = " + prediction.Value);
}
}
} while (true);
engine.SaveDB(EngineDb);
}
public ICalculateEngine GetCalculateEngine()
{
PredictionEngine predicEngine = new PredictionEngine(m_AppContext);
return predicEngine;
}
public void ReturnPredictionEngine(PredictionEngine <TInput, TPrediction> engine)
{
ReturnPredictionEngine(string.Empty, engine);
}
static void Main(string[] args)
{
MLContext mlContext = new MLContext();
// Load Data
IDataView dataView = mlContext.Data.LoadFromTextFile <SentimentData>(_dataPath, hasHeader: false);
TrainTestData splitDataView = mlContext.Data.TrainTestSplit(dataView, testFraction: 0.2);
// BuildAndTrainModel
var estimator = mlContext.Transforms.Text
.FeaturizeText(outputColumnName: "Features", inputColumnName: nameof(SentimentData.SentimentText))
.Append(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(labelColumnName: "Label", featureColumnName: "Features", maximumNumberOfIterations: 100));
Console.WriteLine("=============== Create and Train the Model ===============");
var model = estimator.Fit(splitDataView.TrainSet);
Console.WriteLine("=============== End of training ===============");
Console.WriteLine();
// Evaluate
Console.WriteLine("=============== Evaluating Model accuracy with Test data===============");
IDataView predictions = model.Transform(splitDataView.TestSet);
CalibratedBinaryClassificationMetrics metrics = mlContext.BinaryClassification.Evaluate(predictions, "Label");
Console.WriteLine();
Console.WriteLine("Model quality metrics evaluation");
Console.WriteLine("--------------------------------");
Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
Console.WriteLine($"Auc: {metrics.AreaUnderRocCurve:P2}");
Console.WriteLine($"F1Score: {metrics.F1Score:P2}");
Console.WriteLine("=============== End of model evaluation ===============");
// UseModelWithSingleItem
PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = mlContext.Model.CreatePredictionEngine <SentimentData, SentimentPrediction>(model);
SentimentData sampleStatement = new SentimentData
{
SentimentText = "이 영화 정말 재미없어요"
};
var resultPrediction = predictionFunction.Predict(sampleStatement);
Console.WriteLine();
Console.WriteLine("=============== Prediction Test of model with a single sample and test dataset===============");
Console.WriteLine();
Console.WriteLine($"Sentiment: {resultPrediction.SentimentText} | Prediction: {(Convert.ToBoolean(resultPrediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultPrediction.Probability} ");
Console.WriteLine("=============== End of Predictions ===============");
Console.WriteLine();
// UseModelWithBatchItems
IEnumerable <SentimentData> sentiments = new[]
{
new SentimentData {
SentimentText = "지루한 영화에요"
},
new SentimentData {
SentimentText = "이거 정말 최고에요!"
},
new SentimentData {
SentimentText = "올해의 영화로 손꼽고 싶군요"
}
};
IDataView batchComments = mlContext.Data.LoadFromEnumerable(sentiments);
predictions = model.Transform(batchComments);
// Use model to predict whether comment data is Positive (1) or Negative (0).
IEnumerable <SentimentPrediction> predictedResults = mlContext.Data.CreateEnumerable <SentimentPrediction>(predictions, reuseRowObject: false);
Console.WriteLine();
Console.WriteLine("=============== Prediction Test of loaded model with multiple samples ===============");
foreach (SentimentPrediction prediction in predictedResults)
{
Console.WriteLine($"Sentiment: {prediction.SentimentText} | Prediction:{ (Convert.ToBoolean(prediction.Prediction) ? "Positive" : "Negative")} | Probability:{ prediction.Probability}");
}
Console.WriteLine("=============== End of predictions ===============");
}
public StudentsController(PredictionEngine <StudentTrainingModel, StudentPredictionModel> predictionEngine,
IConfiguration configuration)
{
this.predictionEngine = predictionEngine;
this.configuration = configuration;
}
private static void PredictSimple(AgeRange example, PredictionEngine <AgeRange, AgeRangePrediction> engine)
{
var prediction = engine.Predict(example);
Console.WriteLine($"Name: {example.Name}\t Age: {example.Age:00}\t Gender: {example.Gender}\t Original Label: {example.Label}\t>> Predicted Label: {prediction.PredictedLabel}");
}
public IEnumerable <OutputRow> PredictMany(IEnumerable <InputRow> data)
{
predictionEngine = predictionEngine ?? mLContext.Model.CreatePredictionEngine <InputRow, OutputRow>(transformer);
return(data.Select(d => predictionEngine.Predict(d)));
}
public MyPredictionEngine(IHostEnvironment env, ITransformer pipe)
{
IDataView dv = env.CreateDataView(new TSrc[0]);
_engine = env.CreatePredictionEngine <TSrc, TDst>(pipe.Transform(dv));
}
public static async Task <Dictionary <int, double> > Run(Request request)
{
CreateFile(request.ItemCustomersList);
//STEP 1: Create MLContext to be shared across the model creation workflow objects
var mlContext = new MLContext();
//STEP 2: Read the trained data using TextLoader by defining the schema for reading the product co-purchase dataset
// Do remember to replace amazon0302.txt with dataset from https://snap.stanford.edu/data/amazon0302.html
IDataView dataView = mlContext.Data.LoadFromTextFile(path: TrainingDataLocation,
new[]
{
new TextLoader.Column("Label", DataKind.Single, 0),
new TextLoader.Column(nameof(ProductEntry.CustomerNumber), DataKind.UInt32, new [] { new TextLoader.Range(0) }, new KeyCount(10000)),
new TextLoader.Column(nameof(ProductEntry.RelatedItemId), DataKind.UInt32, new [] { new TextLoader.Range(1) }, new KeyCount(10000))
},
hasHeader: true);
//STEP 3: Your data is already encoded so all you need to do is specify options for MatrxiFactorizationTrainer with a few extra hyperparameters
// LossFunction, Alpa, Lambda and a few others like K and C as shown below and call the trainer.
var options = new MatrixFactorizationTrainer.Options
{
MatrixColumnIndexColumnName = nameof(ProductEntry.CustomerNumber),
MatrixRowIndexColumnName = nameof(ProductEntry.RelatedItemId),
LabelColumnName = "Label",
LossFunction = MatrixFactorizationTrainer.LossFunctionType.SquareLossOneClass,
Alpha = 0.01,
Lambda = 0.025
};
// For better results use the following parameters
//options.K = 100;
//options.C = 0.00001;
//Step 4: Call the MatrixFactorization trainer by passing options.
MatrixFactorizationTrainer est = mlContext.Recommendation().Trainers.MatrixFactorization(options);
//STEP 5: Train the model fitting to the DataSet
//Please add Amazon0302.txt dataset from https://snap.stanford.edu/data/amazon0302.html to Data folder if FileNotFoundException is thrown.
ITransformer model = est.Fit(dataView);
//STEP 6: Create prediction engine and predict the score for Product 63 being co-purchased with Product 3.
// The higher the score the higher the probability for this particular productID being co-purchased
PredictionEngine <ProductEntry, PredictionScore> predictionEngine = mlContext.Model.CreatePredictionEngine <ProductEntry, PredictionScore>(model);
var scores = new Dictionary <int, double>();
foreach (int itemId in request.AllItemsIds)
{
var entry = new ProductEntry
{
CustomerNumber = (uint)request.CustomerNumber,
RelatedItemId = (uint)itemId
};
PredictionScore predictionScore = predictionEngine.Predict(entry);
double finalScore = Math.Round(predictionScore.Score, 3);
scores.Add(itemId, finalScore);
}
scores = scores.ToDictionary(pair => pair.Key, pair => pair.Value);
return(scores);
}
public CodeRecognitionModelController(IOptions <CustomBotOptions <RecyclingBot> > options)
{
_predictionEngine = LoadPredictionEngine(options.Value.CodeRecognitionModelFileName);
}
/// <inheritdoc /> public override bool Return(PredictionEngine <TData, TPrediction> obj) => true;
public SentimentController(PredictionEngine <SentimentData, SentimentPrediction> predictionEnginePool)
{
PredictionEngine = predictionEnginePool;
}
static async Task ClusteringExample(bool train = true)
{
var bestAlg = string.Empty;
double avgdist = double.MaxValue;
var mlContext = new MLContext();
var sqlConnection = $@"Server = localhost;database = Local;user = sa;password = sa";
var traindata = await Utilities.SQL.SQLServer.ExecuteReaderAsync <Iris>(sqlConnection, "SELECT * FROM [iris] ORDER BY NEWID()");
var trainSize = (int)(traindata.Count() * 0.8);
var testdata = traindata.Skip(trainSize).ToList();
traindata = traindata.Take(trainSize).ToList();
var algorithms = new Dictionary <string, Func <IEnumerable <Iris>, Action <ITransformer>, PredictionEngine <Iris, IrisClustering> > >()
{
{ "KMeans", (data, action) => Clustering.KMeans <Iris, IrisClustering>(data, 3, additionModelAction: action) },
};
foreach (var algorithm in algorithms)
{
PredictionEngine <Iris, IrisClustering> engine = default;
ITransformer model = default;
var path = $@"Clustering_{algorithm.Key}.zip";
if (File.Exists(path) && !train)
{
model = MachineLearning.Global.LoadModel(path);
engine = mlContext.Model.CreatePredictionEngine <Iris, IrisClustering>(model);
}
else
{
engine = algorithm.Value(traindata, (mdl) =>
{
model = mdl;
});
}
MachineLearning.Global.SaveModel(model, path);
MachineLearning.ConsoleHelper.ConsoleWriteHeader($@"Evaluate metrics for {algorithm.Key} algorithm.");
var dataframe = new MLContext().Data.LoadFromEnumerable(testdata);
var metrics = Metrics.EvaluateClusteringMetrics(model, dataframe);
foreach (var prop in metrics.GetType().GetProperties())
{
Console.WriteLine($@"{prop.Name} : {prop.GetValue(metrics)}");
}
if (metrics.AverageDistance < avgdist)
{
avgdist = metrics.AverageDistance;
bestAlg = algorithm.Key;
}
var predictedData = new List <IrisClustering>();
foreach (var t in testdata)
{
var temp = t.Label;
var predict = engine.Predict(t);
predictedData.Add(predict);
Console.WriteLine(string.Format(@"Cluster ID : {0,5}", predict.Predicted_cluster));
}
VisualizeClustering(predictedData, "clustering.svg");
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($@"Best algorithm based-on Average Distance : {bestAlg}");
Console.ForegroundColor = ConsoleColor.White;
}
static async Task MulticlassClassificationExample(bool train = true)
{
var bestAlg = string.Empty;
double logLoss = double.MaxValue;
var mlContext = new MLContext();
var sqlConnection = $@"Server = localhost;database = Local;user = sa;password = sa";
var traindata = await Utilities.SQL.SQLServer.ExecuteReaderAsync <Iris>(sqlConnection, "SELECT * FROM [Iris] ORDER BY NEWID()");
var testdata = traindata.Take(20);
var algorithms = new Dictionary <string, Func <IEnumerable <Iris>, Action <ITransformer>, PredictionEngine <Iris, IrisClassification> > >()
{
{ "SdcaMaximumEntropy", (data, action) => MulticlassClassfication.SdcaMaximumEntropy <Iris, IrisClassification>(data, additionModelAction: action) },
{ "LbfgsMaximumEntropy", (data, action) => MulticlassClassfication.LbfgsMaximumEntropy <Iris, IrisClassification>(data, additionModelAction: action) },
{ "NaiveBayes", (data, action) => MulticlassClassfication.NaiveBayes <Iris, IrisClassification>(data, additionModelAction: action) },
};
foreach (var algorithm in algorithms)
{
PredictionEngine <Iris, IrisClassification> engine = default;
ITransformer model = default;
var path = $@"MClassification_{algorithm.Key}.zip";
if (File.Exists(path) && !train)
{
model = MachineLearning.Global.LoadModel(path);
engine = mlContext.Model.CreatePredictionEngine <Iris, IrisClassification>(model);
}
else
{
engine = algorithm.Value(traindata, (mdl) =>
{
model = mdl;
});
}
MachineLearning.Global.SaveModel(model, $@"Multiclass_{algorithm.Key}.zip");
MachineLearning.ConsoleHelper.ConsoleWriteHeader($@"Evaluate metrics for {algorithm.Key} algorithm.");
var metrics = Metrics.EvaluateMulticlassClassificationMetrics(model, mlContext.Data.LoadFromEnumerable(testdata), labelColumnName: nameof(Iris.Label));
foreach (var prop in metrics.GetType().GetProperties())
{
Console.WriteLine($@"{prop.Name} : {prop.GetValue(metrics)}");
}
if (metrics.LogLoss < logLoss)
{
logLoss = metrics.LogLoss;
bestAlg = algorithm.Key;
}
List <IrisClassification> irisClassifications = new List <IrisClassification>();
foreach (var t in testdata)
{
var predict = engine.Predict(t);
irisClassifications.Add(predict);
Console.WriteLine(string.Format(@"Actual : {0,5} / Predict {1,5} {2}", t.Label, predict.Predicted_result, predict.ComparePrediction(t)));
}
//VisualizeMulticlassClassification(algorithm.Key, testdata, irisClassifications, $"{algorithm.Key}_clsf.svg");
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($@"Best algorithm based-on Log Loss : {bestAlg}");
Console.ForegroundColor = ConsoleColor.White;
}
static async Task BinaryClassifier(bool train = true)
{
var sqlConnection = $@"Server = localhost;database = Local;user = sa;password = sa";
var traindata = await Utilities.SQL.SQLServer.ExecuteReaderAsync <HeartData>(sqlConnection, $@"SELECT * FROM HeartTraining");
var testdata = await Utilities.SQL.SQLServer.ExecuteReaderAsync <HeartData>(sqlConnection, $@"SELECT * FROM HeartTest");
double accuracy = double.MinValue;
string bestAlg = string.Empty;
var mlContext = new MLContext();
var algorithms = new Dictionary <string, Func <IEnumerable <HeartData>, Action <ITransformer>, PredictionEngine <HeartData, HeartPredict> > >()
{
{ "FastTree", (data, action) => BinaryClassification.FastTree <HeartData, HeartPredict>(data, additionModelAction: action) },
{ "FastForest", (data, action) => BinaryClassification.FastForest <HeartData, HeartPredict>(data, additionModelAction: action) },
{ "SdcaLogisticRegression", (data, action) => BinaryClassification.SdcaLogisticRegression <HeartData, HeartPredict>(data, additionModelAction: action) },
};
foreach (var algorithm in algorithms)
{
PredictionEngine <HeartData, HeartPredict> engine = default;
ITransformer model = default;
var path = $@"BClassification_{algorithm.Key}.zip";
if (File.Exists(path) && !train)
{
model = MachineLearning.Global.LoadModel(path);
engine = mlContext.Model.CreatePredictionEngine <HeartData, HeartPredict>(model);
}
else
{
engine = algorithm.Value(traindata, (mdl) =>
{
model = mdl;
});
}
MachineLearning.Global.SaveModel(model, $@"Binaryclass_{algorithm.Key}.zip");
MachineLearning.ConsoleHelper.ConsoleWriteHeader($@"Evaluate metrics for {algorithm.Key} algorithm.");
try
{
var metrics = Metrics.EvaluateBinaryClassificationMetrics(model, mlContext.Data.LoadFromEnumerable(testdata), labelColumnName: nameof(HeartData.Label));
foreach (var prop in metrics.GetType().GetProperties())
{
Console.WriteLine($@"{prop.Name} : {prop.GetValue(metrics)}");
}
if (metrics.Accuracy > accuracy)
{
accuracy = metrics.Accuracy;
bestAlg = algorithm.Key;
}
}
catch (Exception e)
{
Console.WriteLine($"Unable to evaluate metrics : {e.Message}");
}
foreach (var t in testdata)
{
var predict = engine.Predict(t);
Console.WriteLine(string.Format(@"Actual {0,5} / Predict {1,5} with prob of {2,5}", t.Label, predict.Prediction, predict.Probability));
}
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($@"Best algorithm based-on accuracy : {bestAlg}");
Console.ForegroundColor = ConsoleColor.White;
}
public void BuildModel()
{
//основная модель - поиск команд
MLContext _mlContext = new MLContext();
String FilePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
IDataView trainingDataView = _mlContext.Data.LoadFromTextFile <CommandsData>(FilePath + "\\train_commands.csv", separatorChar: ';', hasHeader: false);
// Create an IEnumerable from IDataView
IEnumerable <CommandsData> trainingDataViewEnumerable = _mlContext.Data.CreateEnumerable <CommandsData>(trainingDataView, reuseRowObject: true);
var r_text_list = trainingDataViewEnumerable.Select(r => r.TextR).ToList();
//создаем словарь (токены) из слов для команд
char[] separators = new char[] { ' ', '.' };
string string_words = string.Join(" ", r_text_list.ToArray());
string[] test_subs = string_words.Split(separators, StringSplitOptions.RemoveEmptyEntries);
List <string> words_command_list = new List <string>(test_subs);
CommandsTokens = words_command_list.Distinct().ToList();
//сохраняем список токенов в файл
System.IO.File.WriteAllLines(FilePath + "\\CommandsTokens.txt", CommandsTokens);
//заполним также список всех команд (label) в отдельный список.
CommandsList = trainingDataViewEnumerable.Select(r => r.Command).ToList().Distinct().ToList();
//сохраняем список команд в файл
System.IO.File.WriteAllLines(FilePath + "\\Commands.txt", CommandsList);
var pipeline = _mlContext.Transforms.Conversion.MapValueToKey(inputColumnName: "Command", outputColumnName: "Label")
.Append(_mlContext.Transforms.Text.FeaturizeText(inputColumnName: "TextR", outputColumnName: "Features"))
.AppendCacheCheckpoint(_mlContext);
var trainingPipeline = pipeline.Append(_mlContext.MulticlassClassification.Trainers.SdcaMaximumEntropy("Label", "Features"))
.Append(_mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel"));
ITransformer _trainedModel = trainingPipeline.Fit(trainingDataView);
_predEngine = _mlContext.Model.CreatePredictionEngine <CommandsData, IssuePrediction>(_trainedModel);
// Save model
_mlContext.Model.Save(_trainedModel, trainingDataView.Schema, FilePath + "\\model_main_cls.zip");
//вспомогательная модель - отбор по периодам
MLContext _mlContext_periods = new MLContext();
trainingDataView = _mlContext.Data.LoadFromTextFile <CommandsData>(FilePath + "\\train_periods.csv", separatorChar: ';', hasHeader: false);
// Create an IEnumerable from IDataView
trainingDataViewEnumerable = _mlContext_periods.Data.CreateEnumerable <CommandsData>(trainingDataView, reuseRowObject: true);
r_text_list = trainingDataViewEnumerable.Select(r => r.TextR).ToList();
string_words = string.Join(" ", r_text_list.ToArray());
test_subs = string_words.Split(separators, StringSplitOptions.RemoveEmptyEntries);
words_command_list = new List <string>(test_subs);
PeriodsTokens = words_command_list.Distinct().ToList();
//сохраняем токены в файл
System.IO.File.WriteAllLines(FilePath + "\\PeriodsTokens.txt", PeriodsTokens);
var pipeline_periods = _mlContext_periods.Transforms.Conversion.MapValueToKey(inputColumnName: "Command", outputColumnName: "Label")
.Append(_mlContext_periods.Transforms.Text.FeaturizeText(inputColumnName: "TextR", outputColumnName: "Features"))
.AppendCacheCheckpoint(_mlContext_periods);
var trainingPipelinePeriods = pipeline_periods.Append(_mlContext_periods.MulticlassClassification.Trainers.SdcaMaximumEntropy("Label", "Features"))
.Append(_mlContext_periods.Transforms.Conversion.MapKeyToValue("PredictedLabel"));
ITransformer _trainedModel_periods = trainingPipelinePeriods.Fit(trainingDataView);
_predEngine_periods = _mlContext_periods.Model.CreatePredictionEngine <CommandsData, IssuePrediction>(_trainedModel_periods);
// Save model
_mlContext.Model.Save(_trainedModel_periods, trainingDataView.Schema, FilePath + "\\model_periods_cls.zip");
}
protected virtual IEnumerable <TransformedArticleData> GetTransformedArticles(PredictionEngine <ArticleData, TransformedArticleData> predictionEngine, string personId, string language)
{
foreach (var articles in _articleQueryRepository.GetAll(language))
{
foreach (var article in articles)
{
var transformedText = predictionEngine.Predict(ArticleData.Transform(article));
yield return(transformedText);
}
}
}
public SignPredicition()
{
mLContext = new MLContext(seed: 0);
trainedModel = mLContext.Model.Load(Environment.CurrentDirectory + "\\model.zip", out schema);
prediction = mLContext.Model.CreatePredictionEngine <InputData, OutPutData>(trainedModel);
}
public static PredictionData GetMLBBaseballBatterSeasonPredictions(string algorithmName, MLBBaseballBatter mLBBaseballBatter, List <MLBBaseballBatter> selectedBatterSeasons)
{
// Object to return
var predictionData = new PredictionData();
var _predictionEngineInductedToHallOfFameKey = $"Inducted-{algorithmName}";
var _predictionEngineOnHallOfFameBallotKey = $"OnHallOfFameBallot-{algorithmName}";
var mlbBaseballBatterSeasonPredictions = new List <MLBBaseballBatterSeasonPrediction>();
var chartData = new List <PredictionChartData>();
var algorithNamesForEnsemble = new List <string> {
"FastTree", "GeneralizedAdditiveModels", "LightGbm",
"LogisticRegression", "StochasticGradientDescentCalibrated"
};
// If algorithm is selected that does not return probabilities add it
var exists = algorithNamesForEnsemble.Exists(a => a.Contains(algorithmName));
if ((!exists) && (algorithmName != "StackedEnsemble"))
{
algorithNamesForEnsemble.Add(algorithmName);
}
for (int i = 0; i != mLBBaseballBatter.YearsPlayed; i++)
{
var season = i + 1;
var batterSeason = selectedBatterSeasons.Where(s => Convert.ToInt32(s.YearsPlayed) == season).First();
MLBBaseballBatterSeasonPrediction seasonPrediction;
var probabilitiesInducted = new List <AlgorithmPrediction>();
var probabilitiesOnHallOfFameBallot = new List <AlgorithmPrediction>();
foreach (var algorithmNameEnsemble in algorithNamesForEnsemble)
{
PredictionEngine <MLBBaseballBatter, MLBHOFPrediction> _predictionEngineInductedToHallOfFameEnsemble =
Util.GetPredictionEngine(mlContext, "InductedToHallOfFame", algorithmNameEnsemble);
PredictionEngine <MLBBaseballBatter, MLBHOFPrediction> _predictionEngineOnHallOfFameBallotEnsemble =
Util.GetPredictionEngine(mlContext, "OnHallOfFameBallot", algorithmNameEnsemble);
// Note: Cannot do this in Blazor client-side, Transform method starts multiple threads which is not supported
//var modelTest = Util.GetModel(mlContext, "OnHallOfFameBallot", algorithmNameEnsemble);
//var seasonsView = mlContext.Data.LoadFromEnumerable(selectedBatterSeasons);
//var preview = modelTest.Transform(seasonsView).Preview();
var onHallOfFameBallotPredictionEnsemble = _predictionEngineOnHallOfFameBallotEnsemble.Predict(batterSeason);
var inductedToHallOfFamePredictionEnsemble = _predictionEngineInductedToHallOfFameEnsemble.Predict(batterSeason);
probabilitiesInducted.Add(
new AlgorithmPrediction
{
AlgorithmName = algorithmNameEnsemble,
Prediction = inductedToHallOfFamePredictionEnsemble.Prediction,
Probability = inductedToHallOfFamePredictionEnsemble.Probability
});
probabilitiesOnHallOfFameBallot.Add(
new AlgorithmPrediction
{
AlgorithmName = algorithmNameEnsemble,
Prediction = onHallOfFameBallotPredictionEnsemble.Prediction,
Probability = onHallOfFameBallotPredictionEnsemble.Probability
});
// Only add probabilities for algorithms that return probabilities
if (algorithmName == "FastTree" || algorithmName == "GeneralizedAdditiveModels" || algorithmName == "LightGbm" || algorithmName == "LogisticRegression" ||
algorithmName == "StochasticGradientDescentCalibrated" || algorithmName == "StackedEnsemble")
{
chartData.Add(new PredictionChartData
{
Algorithm = algorithmNameEnsemble,
InductedToHallOfFameProbability = inductedToHallOfFamePredictionEnsemble.Probability,
OnHallOfFameBallotProbability = onHallOfFameBallotPredictionEnsemble.Probability,
SeasonPlayed = season
});
}
} // EOF Foreach Algorithm Ensemble
if (algorithmName == "StackedEnsemble")
{
// Average out predictions for ensemble
var probabilityInducted = probabilitiesInducted.Select(a => a.Probability).Sum() / 5;
var probabilityOnHallOfFameBallot = probabilitiesOnHallOfFameBallot.Select(a => a.Probability).Sum() / 5;
seasonPrediction = new MLBBaseballBatterSeasonPrediction
{
SeasonNumber = season,
FullPlayerName = mLBBaseballBatter.FullPlayerName,
InductedToHallOfFamePrediction = (probabilityInducted > 0.5f) ? true : false,
InductedToHallOfFameProbability = probabilityInducted,
OnHallOfFameBallotPrediction = (probabilityOnHallOfFameBallot > 0.5f) ? true : false,
OnHallOfFameBallotProbability = probabilityOnHallOfFameBallot
};
}
else
{
// Average out predictions for ensemble
var probabilityInducted = probabilitiesInducted.Where(a => a.AlgorithmName == algorithmName).FirstOrDefault()?.Probability ?? 0f;
var probabilityOnHallOfFameBallot = probabilitiesOnHallOfFameBallot.Where(a => a.AlgorithmName == algorithmName).FirstOrDefault()?.Probability ?? 0f;
seasonPrediction = new MLBBaseballBatterSeasonPrediction
{
SeasonNumber = season,
FullPlayerName = mLBBaseballBatter.FullPlayerName,
InductedToHallOfFamePrediction = probabilitiesInducted.Where(a => a.AlgorithmName == algorithmName).FirstOrDefault().Prediction,
InductedToHallOfFameProbability = probabilityInducted,
OnHallOfFameBallotPrediction = probabilitiesOnHallOfFameBallot.Where(a => a.AlgorithmName == algorithmName).FirstOrDefault().Prediction,
OnHallOfFameBallotProbability = probabilityOnHallOfFameBallot
};
}
seasonPrediction.InductedToHallOfFameProbabilityLabel = (seasonPrediction.InductedToHallOfFameProbability == 0f) ? "N/A" :
Math.Round(seasonPrediction.InductedToHallOfFameProbability, 6, MidpointRounding.AwayFromZero).ToString("0.000");
seasonPrediction.OnHallOfFameBallotProbabilityLabel = (seasonPrediction.OnHallOfFameBallotProbability == 0f) ? "N/A" :
Math.Round(seasonPrediction.OnHallOfFameBallotProbability, 6, MidpointRounding.AwayFromZero).ToString("0.000");
mlbBaseballBatterSeasonPredictions.Add(seasonPrediction);
// Add StackedEnsemble always to the ChartData
chartData.Add(new PredictionChartData
{
Algorithm = "StackedEnsemble",
InductedToHallOfFameProbability = seasonPrediction.InductedToHallOfFameProbability,
OnHallOfFameBallotProbability = seasonPrediction.OnHallOfFameBallotProbability,
SeasonPlayed = season
});
//}
// Get the min/max for each season
var chardDataMin =
chartData
.GroupBy(c => new
{
c.SeasonPlayed
})
.Select(gcs => new PredictionChartDataMinMax()
{
Algorithm = "OnHallOfFameBallot",
SeasonPlayed = gcs.Key.SeasonPlayed,
Min = gcs.Min(g => g.OnHallOfFameBallotProbability),
Max = gcs.Max(g => g.OnHallOfFameBallotProbability)
}).ToList();
var chardDataMax =
chartData
.GroupBy(c => new
{
c.SeasonPlayed
})
.Select(gcs => new PredictionChartDataMinMax()
{
Algorithm = "InductedToHallOfFame",
SeasonPlayed = gcs.Key.SeasonPlayed,
Min = gcs.Min(g => g.InductedToHallOfFameProbability),
Max = gcs.Max(g => g.InductedToHallOfFameProbability)
}).ToList();
predictionData.ChartData = chartData;
chardDataMin.AddRange(chardDataMax);
predictionData.ChartDataMinMax = chardDataMin;
}
predictionData.MLBBaseballBatterSeasonPredictions = mlbBaseballBatterSeasonPredictions;
return(predictionData);
}
public DeseasePrediction(IHttpClientFactory httpClientFactory, MLContext mlContext, PredictionEngine <HeartData, HeartPrediction> predictionEngine)
{
this.httpClient = httpClientFactory.CreateClient();
this.mlContext = mlContext;
this.predictionEngine = predictionEngine;
}
public OutputRow Predict(InputRow data)
{
predictionEngine = predictionEngine ?? mLContext.Model.CreatePredictionEngine <InputRow, OutputRow>(transformer);
return(predictionEngine.Predict(data));
}
static void Main(string[] args)
{
try
{
// Create MLContext
MLContext mlContext = new MLContext();
// Load Trained Model
DataViewSchema predictionPipelineSchema;
ITransformer predictionPipeline = mlContext.Model.Load("../../../MLModels/SentimentModel.zip", out predictionPipelineSchema);
// Create PredictionEngines
PredictionEngine <SentimentIssue, SentimentPrediction> predictionEngine = mlContext.Model.CreatePredictionEngine <SentimentIssue, SentimentPrediction>(predictionPipeline);
// Input Data (single)
SentimentIssue inputData = new SentimentIssue
{
Text = "I love this movie!"
};
// Get Prediction
SentimentPrediction prediction = predictionEngine.Predict(inputData);
Console.WriteLine($"=============== Single Prediction ===============");
Console.WriteLine($"Text: {inputData.Text} | Prediction: {(Convert.ToBoolean(prediction.Prediction) ? "Toxic" : "Non Toxic")} sentiment | Probability of being toxic: {prediction.Probability} ");
Console.WriteLine($"================End of Process.Hit any key to exit==================================");
// input data multiple
SentimentIssue[] inputArray = new SentimentIssue[]
{
new SentimentIssue
{
Text = "I love this movie!"
},
new SentimentIssue
{
Text = "F*****g good!"
},
};
//Load Data
IDataView data = mlContext.Data.LoadFromEnumerable <SentimentIssue>(inputArray);
// Predicted Data
IDataView predictions = predictionPipeline.Transform(data);
// Create an IEnumerable of prediction objects from IDataView
IEnumerable <SentimentPrediction> dataEnumerable =
mlContext.Data.CreateEnumerable <SentimentPrediction>(predictions, reuseRowObject: true);
// Iterate over each row
for (int i = 0; i < inputArray.Length; i++)
{
string input = inputArray[i].Text;
SentimentPrediction item = dataEnumerable.ElementAt(i);
// display
Console.WriteLine($"Text: {input} | Prediction: {(Convert.ToBoolean(item.Prediction) ? "Toxic" : "Non Toxic")} sentiment | Probability of being toxic: {item.Probability} ");
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
Console.ReadLine();
}
public static void SetupEngine()
{
mlContext = new MLContext();
var basePath = BotSettings.LearningDataSetPath;
// var filePath = basePath + "SMSSpamCollection.csv";
// var filePath = basePath + "SpamCollection.csv";
Log.Information("Loading dataset.");
// Specify the schema for spam data and read it into DataView.
// var data = mlContext.Data.LoadFromTextFile<SpamInput>(filePath,
// hasHeader: true, separatorChar: '\t',allowQuoting:true);
var dataSet = new Query("words_learning")
.SelectRaw("label Label, message Message")
.ExecForMysql(true)
.Get <LearnCsv>();
Log.Debug("Load DataSet {V} row(s)", dataSet.Count());
var data = mlContext.Data.LoadFromEnumerable(dataSet);
Log.Information("Creating pipelines.");
// Data process configuration with pipeline data transformations
var dataProcessPipeline = mlContext.Transforms.Conversion
.MapValueToKey("Label", "Label")
.Append(mlContext.Transforms.Text.FeaturizeText("FeaturesText",
new TextFeaturizingEstimator.Options
{
WordFeatureExtractor = new WordBagEstimator.Options
{
NgramLength = 2, UseAllLengths = true
},
CharFeatureExtractor = new WordBagEstimator.Options
{
NgramLength = 3, UseAllLengths = false
}
}, "Message"))
.Append(mlContext.Transforms.CopyColumns("Features", "FeaturesText"))
.Append(mlContext.Transforms.NormalizeLpNorm("Features", "Features"))
.AppendCacheCheckpoint(mlContext);
Log.Information("Set the training algorithm");
var trainer = mlContext.MulticlassClassification
.Trainers.OneVersusAll(mlContext.BinaryClassification
.Trainers.AveragedPerceptron(labelColumnName: "Label",
numberOfIterations: 10, featureColumnName: "Features"), labelColumnName: "Label")
.Append(mlContext.Transforms
.Conversion.MapKeyToValue("PredictedLabel", "PredictedLabel"));
var trainingPipeLine = dataProcessPipeline.Append(trainer);
Log.Information("Evaluate the model using cross-validation.");
// Cross-validation splits our dataset into 'folds', trains a model on some folds and
// evaluates it on the remaining fold. We are using 5 folds so we get back 5 sets of scores.
// Let's compute the average AUC, which should be between 0.5 and 1 (higher is better).
Log.Information("=== Cross-validating to get model's accuracy metrics");
var crossValidationResults =
mlContext.MulticlassClassification.CrossValidate(data: data, estimator: trainingPipeLine,
numberOfFolds: 5);
Log.Information("Trainer: {V}", trainer.ToJson(true));
Log.Information("Starting train a model");
// Now let's train a model on the full dataset to help us get better results
var model = trainingPipeLine.Fit(data);
Log.Information("Create a PredictionFunction from our model");
var predictor = mlContext.Model.CreatePredictionEngine <SpamInput, SpamPrediction>(model);
SpamEngine = predictor;
Console.WriteLine("=============== Predictions for below data===============");
// Test a few examples
ClassifyMessage(predictor, "That's a great idea. It should work.");
ClassifyMessage(predictor, "free medicine winner! congratulations");
ClassifyMessage(predictor, "Yes we should meet over the weekend!");
ClassifyMessage(predictor, "you win pills and free entry vouchers");
ClassifyMessage(predictor, "Lorem ipsum dolor sit amet");
}
public ClassificationController(PredictionEngine <ModelInput, ModelOutput> predictionEngine)
{
_predictionEngine = predictionEngine;
}
private void GenPredictors()
{
predictorX = mlContext.Model.CreatePredictionEngine <RegressionRecord, RegressionPrediction>(estimatorX);
predictorY = mlContext.Model.CreatePredictionEngine <RegressionRecord, RegressionPrediction>(estimatorY);
}
public YoloObjectDetector(YoloPostProcessor postProcessor, YoloPredictionEngineFactory factory)
{
_predictionEngine = factory.BuildPredictionEngine();
_postProcessor = postProcessor;
}
private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
{
PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = mlContext.Model.CreatePredictionEngine <SentimentData, SentimentPrediction>(model);
}
public ModelOutputPredictor(string modelFile)
{
_mlContext = new MLContext();
_trainedModel = _mlContext.Model.Load(modelFile, out var inputSchema);
_predictionEngine = _mlContext.Model.CreatePredictionEngine <ModelInput, ModelOutput>(_trainedModel);
}
public SentimentService()
{
var model = _context.Model.Load(File.Open(@".\SentimentModel.zip", FileMode.Open));
_predictionEngine = model.CreatePredictionEngine <Sentiment, Prediction>(_context);
}
public MLPlayer(string modelPath)
{
ITransformer mlModel = mlContext.Model.Load(GetAbsolutePath(modelPath), out DataViewSchema inputSchema);
predictionEngine = mlContext.Model.CreatePredictionEngine <ModelInput, ModelOutput>(mlModel);
}
public LightGbmBlockClassifier(string modelPath)
{
mlContext = new MLContext();
mlModel = mlContext.Model.Load(modelPath, out var modelInputSchema);
predEngine = mlContext.Model.CreatePredictionEngine <BlockFeatures, BlockCategory>(mlModel);
}
public MikLabelerPredictor(PredictionEngine <GitHubIssue, GitHubIssuePrediction> predictionEngine,
PredictionEngine <GitHubPullRequest, GitHubIssuePrediction> prPredictionEngine)
{
_predictionEngine = predictionEngine;
_prPredictionEngine = prPredictionEngine;
}
