/// <summary>
/// Create the template output
/// </summary>
public virtual string TransformText()
{
if (Target == CSharp.GenerateTarget.Cli)
{
CLI_Annotation();
}
else if (Target == CSharp.GenerateTarget.ModelBuilder)
{
MB_Annotation();
}
this.Write("\r\nusing System;\r\nusing System.IO;\r\nusing System.Linq;\r\nusing Microsoft.ML;\r\nusing" +
" ");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".Model;\r\n\r\nnamespace ");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".ConsoleApp\r\n{\r\n class Program\r\n {\r\n //Dataset to use for prediction" +
"s \r\n");
if (string.IsNullOrEmpty(TestDataPath))
{
this.Write(" private const string DATA_FILEPATH = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(TrainDataPath));
this.Write("\";\r\n");
}
else
{
this.Write(" private const string DATA_FILEPATH = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(TestDataPath));
this.Write("\";\r\n");
}
this.Write(@"
static void Main(string[] args)
{
// Create single instance of sample data from first line of dataset for model input
ModelInput sampleData = CreateSingleDataSample(DATA_FILEPATH);
// Make a single prediction on the sample data and print results
var predictionResult = ConsumeModel.Predict(sampleData);
Console.WriteLine(""Using model to make single prediction -- Comparing actual " );
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(" with predicted ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(" from sample data...\\n\\n\");\r\n");
foreach (var label in Features)
{
this.Write("\t\t\tConsole.WriteLine($\"");
this.Write(this.ToStringHelper.ToStringWithCulture(label));
this.Write(": {sampleData.");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(label)));
this.Write("}\");\r\n");
}
if ("BinaryClassification".Equals(TaskType))
{
this.Write("\t\t\tConsole.WriteLine($\"\\n\\nActual ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(": {sampleData.");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write("} \\nPredicted ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(": {predictionResult.Prediction}\\n\\n\");\r\n");
}
else if ("Regression".Equals(TaskType) || "Recommendation".Equals(TaskType))
{
this.Write("\t\t\tConsole.WriteLine($\"\\n\\nActual ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(": {sampleData.");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write("} \\nPredicted ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(": {predictionResult.Score}\\n\\n\");\r\n");
}
else if ("MulticlassClassification".Equals(TaskType))
{
this.Write("\t\t\tConsole.WriteLine($\"\\n\\nActual ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(": {sampleData.");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write("} \\nPredicted ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(" value {predictionResult.Prediction} \\nPredicted ");
this.Write(this.ToStringHelper.ToStringWithCulture(Utils.Normalize(LabelName)));
this.Write(" scores: [{String.Join(\",\", predictionResult.Score)}]\\n\\n\");\r\n");
}
this.Write(@" Console.WriteLine(""=============== End of process, hit any key to finish ==============="");
Console.ReadKey();
}
// Change this code to create your own sample data
#region CreateSingleDataSample
// Method to load single row of dataset to try a single prediction
private static ModelInput CreateSingleDataSample(string dataFilePath)
{
// Create MLContext
MLContext mlContext = new MLContext();
// Load dataset
IDataView dataView = mlContext.Data.LoadFromTextFile<ModelInput>(
path: dataFilePath,
hasHeader : ");
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
this.Write(",\r\n separatorChar : \'");
this.Write(this.ToStringHelper.ToStringWithCulture(Regex.Escape(Separator.ToString())));
this.Write("\',\r\n allowQuoting : ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
this.Write(",\r\n allowSparse: ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
this.Write(@");
// Use first line of dataset as model input
// You can replace this with new test data (hardcoded or from end-user application)
ModelInput sampleForPrediction = mlContext.Data.CreateEnumerable<ModelInput>(dataView, false)
.First();
return sampleForPrediction;
}
#endregion
}
}
");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public virtual string TransformText()
{
if (Target == CSharp.GenerateTarget.Cli)
{
CLI_Annotation();
}
else if (Target == CSharp.GenerateTarget.ModelBuilder)
{
MB_Annotation();
}
this.Write("\r\nusing System;\r\nusing System.Collections.Generic;\r\nusing System.IO;\r\nusing Syste" +
"m.Linq;\r\nusing Microsoft.ML;\r\nusing Microsoft.ML.Data;\r\nusing ");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".Model;\r\n");
this.Write(this.ToStringHelper.ToStringWithCulture(GeneratedUsings));
this.Write("\r\nnamespace ");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".ConsoleApp\r\n{\r\n public static class ModelBuilder\r\n {\r\n private stat" +
"ic string TRAIN_DATA_FILEPATH = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(Path));
this.Write("\";\r\n");
if (!string.IsNullOrEmpty(TestPath))
{
this.Write(" private static string TEST_DATA_FILEPATH = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(TestPath));
this.Write("\";\r\n");
}
this.Write(" private static string MODEL_FILEPATH = @\"../../../../");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(@".Model/MLModel.zip"";
// Create MLContext to be shared across the model creation workflow objects
// Set a random seed for repeatable/deterministic results across multiple trainings.
private static MLContext mlContext = new MLContext(seed: 1);
public static void CreateModel()
{
// Load Data
IDataView trainingDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
path: TRAIN_DATA_FILEPATH,
hasHeader : ");
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
this.Write(",\r\n separatorChar : \'");
this.Write(this.ToStringHelper.ToStringWithCulture(Regex.Escape(Separator.ToString())));
this.Write("\',\r\n allowQuoting : ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
this.Write(",\r\n allowSparse: ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
this.Write(");\r\n\r\n");
if (!string.IsNullOrEmpty(TestPath))
{
this.Write(" IDataView testDataView = mlContext.Data.LoadFromTextFile<ModelInput>(" +
"\r\n path: TEST_DATA_FILEPATH,\r\n " +
" hasHeader : ");
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
this.Write(",\r\n separatorChar : \'");
this.Write(this.ToStringHelper.ToStringWithCulture(Regex.Escape(Separator.ToString())));
this.Write("\',\r\n allowQuoting : ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
this.Write(",\r\n allowSparse: ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
this.Write(");\r\n");
}
this.Write(" // Build training pipeline\r\n IEstimator<ITransformer> trai" +
"ningPipeline = BuildTrainingPipeline(mlContext);\r\n\r\n");
if (string.IsNullOrEmpty(TestPath))
{
this.Write(" // Evaluate quality of Model\r\n Evaluate(mlContext, trainin" +
"gDataView, trainingPipeline);\r\n\r\n");
}
this.Write(" // Train Model\r\n ITransformer mlModel = TrainModel(mlConte" +
"xt, trainingDataView, trainingPipeline);\r\n");
if (!string.IsNullOrEmpty(TestPath))
{
this.Write("\r\n // Evaluate quality of Model\r\n EvaluateModel(mlContext, " +
"mlModel, testDataView);\r\n");
}
this.Write("\r\n // Save model\r\n SaveModel(mlContext, mlModel, MODEL_FILE" +
"PATH, trainingDataView.Schema);\r\n }\r\n\r\n public static IEstimator<I" +
"Transformer> BuildTrainingPipeline(MLContext mlContext)\r\n {\r\n");
if (PreTrainerTransforms.Count > 0)
{
this.Write(" // Data process configuration with pipeline data transformations \r\n " +
" var dataProcessPipeline = ");
for (int i = 0; i < PreTrainerTransforms.Count; i++)
{
if (i > 0)
{
Write("\r\n .Append(");
}
Write("mlContext.Transforms." + PreTrainerTransforms[i]);
if (i > 0)
{
Write(")");
}
}
if (CacheBeforeTrainer)
{
Write("\r\n .AppendCacheCheckpoint(mlContext)");
}
this.Write(";\r\n");
}
this.Write("\r\n // Set the training algorithm \r\n var trainer = mlContext" +
".");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".Trainers.");
this.Write(this.ToStringHelper.ToStringWithCulture(Trainer));
for (int i = 0; i < PostTrainerTransforms.Count; i++)
{
Write("\r\n .Append(");
Write("mlContext.Transforms." + PostTrainerTransforms[i]);
Write(")");
}
this.Write(";\r\n");
if (PreTrainerTransforms.Count > 0)
{
this.Write(" var trainingPipeline = dataProcessPipeline.Append(trainer);\r\n");
}
else
{
this.Write(" var trainingPipeline = trainer;\r\n");
}
this.Write(@"
return trainingPipeline;
}
public static ITransformer TrainModel(MLContext mlContext, IDataView trainingDataView, IEstimator<ITransformer> trainingPipeline)
{
Console.WriteLine(""=============== Training model ==============="");
ITransformer model = trainingPipeline.Fit(trainingDataView);
Console.WriteLine(""=============== End of training process ==============="");
return model;
}
");
if (!string.IsNullOrEmpty(TestPath))
{
this.Write(@" private static void EvaluateModel(MLContext mlContext, ITransformer mlModel, IDataView testDataView)
{
// Evaluate the model and show accuracy stats
Console.WriteLine(""===== Evaluating Model's accuracy with Test data ====="");
IDataView predictions = mlModel.Transform(testDataView);
");
if ("BinaryClassification".Equals(TaskType))
{
this.Write(" var metrics = mlContext.");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".EvaluateNonCalibrated(predictions, \"");
this.Write(this.ToStringHelper.ToStringWithCulture(LabelName));
this.Write("\", \"Score\");\r\n PrintBinaryClassificationMetrics(metrics);\r\n");
}
if ("MulticlassClassification".Equals(TaskType))
{
this.Write(" var metrics = mlContext.");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".Evaluate(predictions, \"");
this.Write(this.ToStringHelper.ToStringWithCulture(LabelName));
this.Write("\", \"Score\");\r\n PrintMulticlassClassificationMetrics(metrics);\r\n");
}
if ("Regression".Equals(TaskType))
{
this.Write(" var metrics = mlContext.");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".Evaluate(predictions, \"");
this.Write(this.ToStringHelper.ToStringWithCulture(LabelName));
this.Write("\", \"Score\");\r\n PrintRegressionMetrics(metrics);\r\n");
}
this.Write(" }\r\n");
}
else
{
this.Write(@" private static void Evaluate(MLContext mlContext, IDataView trainingDataView, IEstimator<ITransformer> trainingPipeline)
{
// Cross-Validate with single dataset (since we don't have two datasets, one for training and for evaluate)
// in order to evaluate and get the model's accuracy metrics
Console.WriteLine(""=============== Cross-validating to get model's accuracy metrics ==============="");
");
if ("BinaryClassification".Equals(TaskType))
{
this.Write(" var crossValidationResults = mlContext.");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".CrossValidateNonCalibrated(trainingDataView, trainingPipeline, numberOfFolds: ");
this.Write(this.ToStringHelper.ToStringWithCulture(Kfolds));
this.Write(", labelColumnName:\"");
this.Write(this.ToStringHelper.ToStringWithCulture(LabelName));
this.Write("\");\r\n PrintBinaryClassificationFoldsAverageMetrics(crossValidationResu" +
"lts);\r\n");
}
if ("MulticlassClassification".Equals(TaskType))
{
this.Write(" var crossValidationResults = mlContext.");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".CrossValidate(trainingDataView, trainingPipeline, numberOfFolds: ");
this.Write(this.ToStringHelper.ToStringWithCulture(Kfolds));
this.Write(", labelColumnName:\"");
this.Write(this.ToStringHelper.ToStringWithCulture(LabelName));
this.Write("\");\r\n PrintMulticlassClassificationFoldsAverageMetrics(crossValidation" +
"Results);\r\n");
}
if ("Regression".Equals(TaskType))
{
this.Write(" var crossValidationResults = mlContext.");
this.Write(this.ToStringHelper.ToStringWithCulture(TaskType));
this.Write(".CrossValidate(trainingDataView, trainingPipeline, numberOfFolds: ");
this.Write(this.ToStringHelper.ToStringWithCulture(Kfolds));
this.Write(", labelColumnName:\"");
this.Write(this.ToStringHelper.ToStringWithCulture(LabelName));
this.Write("\");\r\n PrintRegressionFoldsAverageMetrics(crossValidationResults);\r\n");
}
this.Write(" }\r\n");
}
this.Write(@" private static void SaveModel(MLContext mlContext, ITransformer mlModel, string modelRelativePath, DataViewSchema modelInputSchema)
{
// Save/persist the trained model to a .ZIP file
Console.WriteLine($""=============== Saving the model ==============="");
mlContext.Model.Save(mlModel, modelInputSchema, GetAbsolutePath(modelRelativePath));
Console.WriteLine(""The model is saved to {0}"", GetAbsolutePath(modelRelativePath));
}
public static string GetAbsolutePath(string relativePath)
{
FileInfo _dataRoot = new FileInfo(typeof(Program).Assembly.Location);
string assemblyFolderPath = _dataRoot.Directory.FullName;
string fullPath = Path.Combine(assemblyFolderPath, relativePath);
return fullPath;
}
");
if ("Regression".Equals(TaskType))
{
this.Write(" public static void PrintRegressionMetrics(RegressionMetrics metrics)\r\n " +
" {\r\n Console.WriteLine($\"****************************************" +
"*********\");\r\n Console.WriteLine($\"* Metrics for regression mod" +
"el \");\r\n Console.WriteLine($\"*----------------------------------" +
"--------------\");\r\n Console.WriteLine($\"* LossFn: {metri" +
"cs.LossFunction:0.##}\");\r\n Console.WriteLine($\"* R2 Score: " +
" {metrics.RSquared:0.##}\");\r\n Console.WriteLine($\"* Absolute lo" +
"ss: {metrics.MeanAbsoluteError:#.##}\");\r\n Console.WriteLine($\"* " +
" Squared loss: {metrics.MeanSquaredError:#.##}\");\r\n Console.WriteLin" +
"e($\"* RMS loss: {metrics.RootMeanSquaredError:#.##}\");\r\n C" +
"onsole.WriteLine($\"*************************************************\");\r\n " +
" }\r\n\r\n public static void PrintRegressionFoldsAverageMetrics(IEnumerable<" +
"TrainCatalogBase.CrossValidationResult<RegressionMetrics>> crossValidationResult" +
"s)\r\n {\r\n var L1 = crossValidationResults.Select(r => r.Metrics" +
".MeanAbsoluteError);\r\n var L2 = crossValidationResults.Select(r => r." +
"Metrics.MeanSquaredError);\r\n var RMS = crossValidationResults.Select(" +
"r => r.Metrics.RootMeanSquaredError);\r\n var lossFunction = crossValid" +
"ationResults.Select(r => r.Metrics.LossFunction);\r\n var R2 = crossVal" +
"idationResults.Select(r => r.Metrics.RSquared);\r\n\r\n Console.WriteLine" +
"($\"*****************************************************************************" +
"********************************\");\r\n Console.WriteLine($\"* Met" +
"rics for Regression model \");\r\n Console.WriteLine($\"*-----------" +
"--------------------------------------------------------------------------------" +
"-----------------\");\r\n Console.WriteLine($\"* Average L1 Loss: " +
" {L1.Average():0.###} \");\r\n Console.WriteLine($\"* Average L" +
"2 Loss: {L2.Average():0.###} \");\r\n Console.WriteLine($\"* " +
" Average RMS: {RMS.Average():0.###} \");\r\n Console.WriteLin" +
"e($\"* Average Loss Function: {lossFunction.Average():0.###} \");\r\n " +
" Console.WriteLine($\"* Average R-squared: {R2.Average():0.###} \");" +
"\r\n Console.WriteLine($\"**********************************************" +
"***************************************************************\");\r\n }\r\n");
}
if ("BinaryClassification".Equals(TaskType))
{
this.Write(" public static void PrintBinaryClassificationMetrics(BinaryClassificationM" +
"etrics metrics)\r\n {\r\n Console.WriteLine($\"********************" +
"****************************************\");\r\n Console.WriteLine($\"* " +
" Metrics for binary classification model \");\r\n Console.Write" +
"Line($\"*-----------------------------------------------------------\");\r\n " +
" Console.WriteLine($\"* Accuracy: {metrics.Accuracy:P2}\");\r\n " +
"Console.WriteLine($\"* Auc: {metrics.AreaUnderRocCurve:P2}\");\r\n " +
" Console.WriteLine($\"*******************************************************" +
"*****\");\r\n }\r\n\r\n\r\n public static void PrintBinaryClassificationFol" +
"dsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<BinaryClassi" +
"ficationMetrics>> crossValResults)\r\n {\r\n var metricsInMultiple" +
"Folds = crossValResults.Select(r => r.Metrics);\r\n\r\n var AccuracyValue" +
"s = metricsInMultipleFolds.Select(m => m.Accuracy);\r\n var AccuracyAve" +
"rage = AccuracyValues.Average();\r\n var AccuraciesStdDeviation = Calcu" +
"lateStandardDeviation(AccuracyValues);\r\n var AccuraciesConfidenceInte" +
"rval95 = CalculateConfidenceInterval95(AccuracyValues);\r\n\r\n\r\n Console" +
".WriteLine($\"*******************************************************************" +
"******************************************\");\r\n Console.WriteLine($\"*" +
" Metrics for Binary Classification model \");\r\n Console.Wri" +
"teLine($\"*----------------------------------------------------------------------" +
"--------------------------------------\");\r\n Console.WriteLine($\"* " +
" Average Accuracy: {AccuracyAverage:0.###} - Standard deviation: ({Accurac" +
"iesStdDeviation:#.###}) - Confidence Interval 95%: ({AccuraciesConfidenceInterv" +
"al95:#.###})\");\r\n Console.WriteLine($\"*******************************" +
"******************************************************************************\")" +
";\r\n }\r\n\r\n public static double CalculateStandardDeviation(IEnumera" +
"ble<double> values)\r\n {\r\n double average = values.Average();\r\n" +
" double sumOfSquaresOfDifferences = values.Select(val => (val - avera" +
"ge) * (val - average)).Sum();\r\n double standardDeviation = Math.Sqrt(" +
"sumOfSquaresOfDifferences / (values.Count() - 1));\r\n return standardD" +
"eviation;\r\n }\r\n\r\n public static double CalculateConfidenceInterval" +
"95(IEnumerable<double> values)\r\n {\r\n double confidenceInterval" +
"95 = 1.96 * CalculateStandardDeviation(values) / Math.Sqrt((values.Count() - 1))" +
";\r\n return confidenceInterval95;\r\n }\r\n");
}
if ("MulticlassClassification".Equals(TaskType))
{
this.Write(" public static void PrintMulticlassClassificationMetrics(MulticlassClassif" +
"icationMetrics metrics)\r\n {\r\n Console.WriteLine($\"************" +
"************************************************\");\r\n Console.WriteLi" +
"ne($\"* Metrics for multi-class classification model \");\r\n Consol" +
"e.WriteLine($\"*-----------------------------------------------------------\");\r\n " +
" Console.WriteLine($\" MacroAccuracy = {metrics.MacroAccuracy:0.####" +
"}, a value between 0 and 1, the closer to 1, the better\");\r\n Console." +
"WriteLine($\" MicroAccuracy = {metrics.MicroAccuracy:0.####}, a value between " +
"0 and 1, the closer to 1, the better\");\r\n Console.WriteLine($\" Log" +
"Loss = {metrics.LogLoss:0.####}, the closer to 0, the better\");\r\n for" +
" (int i = 0; i < metrics.PerClassLogLoss.Count; i++)\r\n {\r\n " +
" Console.WriteLine($\" LogLoss for class {i + 1} = {metrics.PerClassLogLos" +
"s[i]:0.####}, the closer to 0, the better\");\r\n }\r\n Console" +
".WriteLine($\"************************************************************\");\r\n " +
" }\r\n\r\n public static void PrintMulticlassClassificationFoldsAverageM" +
"etrics(IEnumerable<TrainCatalogBase.CrossValidationResult<MulticlassClassificati" +
"onMetrics>> crossValResults)\r\n {\r\n var metricsInMultipleFolds " +
"= crossValResults.Select(r => r.Metrics);\r\n\r\n var microAccuracyValues" +
" = metricsInMultipleFolds.Select(m => m.MicroAccuracy);\r\n var microAc" +
"curacyAverage = microAccuracyValues.Average();\r\n var microAccuraciesS" +
"tdDeviation = CalculateStandardDeviation(microAccuracyValues);\r\n var " +
"microAccuraciesConfidenceInterval95 = CalculateConfidenceInterval95(microAccurac" +
"yValues);\r\n\r\n var macroAccuracyValues = metricsInMultipleFolds.Select" +
"(m => m.MacroAccuracy);\r\n var macroAccuracyAverage = macroAccuracyVal" +
"ues.Average();\r\n var macroAccuraciesStdDeviation = CalculateStandardD" +
"eviation(macroAccuracyValues);\r\n var macroAccuraciesConfidenceInterva" +
"l95 = CalculateConfidenceInterval95(macroAccuracyValues);\r\n\r\n var log" +
"LossValues = metricsInMultipleFolds.Select(m => m.LogLoss);\r\n var log" +
"LossAverage = logLossValues.Average();\r\n var logLossStdDeviation = Ca" +
"lculateStandardDeviation(logLossValues);\r\n var logLossConfidenceInter" +
"val95 = CalculateConfidenceInterval95(logLossValues);\r\n\r\n var logLoss" +
"ReductionValues = metricsInMultipleFolds.Select(m => m.LogLossReduction);\r\n " +
" var logLossReductionAverage = logLossReductionValues.Average();\r\n " +
" var logLossReductionStdDeviation = CalculateStandardDeviation(logLossReducti" +
"onValues);\r\n var logLossReductionConfidenceInterval95 = CalculateConf" +
"idenceInterval95(logLossReductionValues);\r\n\r\n Console.WriteLine($\"***" +
"********************************************************************************" +
"**************************\");\r\n Console.WriteLine($\"* Metrics f" +
"or Multi-class Classification model \");\r\n Console.WriteLine($\"*-" +
"--------------------------------------------------------------------------------" +
"---------------------------\");\r\n Console.WriteLine($\"* Average " +
"MicroAccuracy: {microAccuracyAverage:0.###} - Standard deviation: ({microAcc" +
"uraciesStdDeviation:#.###}) - Confidence Interval 95%: ({microAccuraciesConfide" +
"nceInterval95:#.###})\");\r\n Console.WriteLine($\"* Average MacroA" +
"ccuracy: {macroAccuracyAverage:0.###} - Standard deviation: ({macroAccuracie" +
"sStdDeviation:#.###}) - Confidence Interval 95%: ({macroAccuraciesConfidenceInt" +
"erval95:#.###})\");\r\n Console.WriteLine($\"* Average LogLoss: " +
" {logLossAverage:#.###} - Standard deviation: ({logLossStdDeviation:#.###}" +
") - Confidence Interval 95%: ({logLossConfidenceInterval95:#.###})\");\r\n " +
" Console.WriteLine($\"* Average LogLossReduction: {logLossReductionAvera" +
"ge:#.###} - Standard deviation: ({logLossReductionStdDeviation:#.###}) - Confi" +
"dence Interval 95%: ({logLossReductionConfidenceInterval95:#.###})\");\r\n " +
" Console.WriteLine($\"*********************************************************" +
"****************************************************\");\r\n\r\n }\r\n\r\n " +
"public static double CalculateStandardDeviation(IEnumerable<double> values)\r\n " +
" {\r\n double average = values.Average();\r\n double sumOf" +
"SquaresOfDifferences = values.Select(val => (val - average) * (val - average)).S" +
"um();\r\n double standardDeviation = Math.Sqrt(sumOfSquaresOfDifference" +
"s / (values.Count() - 1));\r\n return standardDeviation;\r\n }\r\n\r\n" +
" public static double CalculateConfidenceInterval95(IEnumerable<double> v" +
"alues)\r\n {\r\n double confidenceInterval95 = 1.96 * CalculateSta" +
"ndardDeviation(values) / Math.Sqrt((values.Count() - 1));\r\n return co" +
"nfidenceInterval95;\r\n }\r\n");
}
this.Write(" }\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public virtual string TransformText()
{
if (Target == CSharp.GenerateTarget.Cli)
{
CLI_Annotation();
}
else if (Target == CSharp.GenerateTarget.ModelBuilder)
{
MB_Annotation();
}
this.Write("using System;\r\nusing System.Collections.Generic;\r\nusing System.IO;\r\nusing System." +
"Linq;\r\nusing Microsoft.ML;\r\nusing Microsoft.ML.Data;\r\nusing ");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".Model;\r\nnamespace ");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".ConsoleApp\r\n{\r\n public static class ModelBuilder\r\n {\r\n private stat" +
"ic string TRAIN_DATA_FILEPATH = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(Path));
this.Write("\";\r\n private static string MLNET_MODEL = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(MLNetModelpath));
this.Write("\";\r\n\t\tprivate static string ONNX_MODEL = @\"");
this.Write(this.ToStringHelper.ToStringWithCulture(OnnxModelPath));
this.Write(@""";
// Create MLContext to be shared across the model creation workflow objects
// Set a random seed for repeatable/deterministic results across multiple trainings.
private static MLContext mlContext = new MLContext(seed: 1);
// Training in Azure produces an ONNX model; this method demonstrates creating an ML.NET model (MLModel.zip) from the ONNX model (bestModel.onnx), which you can then use with the ConsumeModel() method to make predictions
public static void CreateMLNetModelFromOnnx()
{
// Load data
IDataView inputDataView = mlContext.Data.LoadFromTextFile<ModelInput>(
path: TRAIN_DATA_FILEPATH,
hasHeader : ");
this.Write(this.ToStringHelper.ToStringWithCulture(HasHeader.ToString().ToLowerInvariant()));
this.Write(",\r\n separatorChar : \'");
this.Write(this.ToStringHelper.ToStringWithCulture(Regex.Escape(Separator.ToString())));
this.Write("\',\r\n allowQuoting : ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowQuoting.ToString().ToLowerInvariant()));
this.Write(",\r\n allowSparse: ");
this.Write(this.ToStringHelper.ToStringWithCulture(AllowSparse.ToString().ToLowerInvariant()));
this.Write(@");
// Create an ML.NET pipeline to score using the ONNX model
// Notice that this pipeline is not trainable because it only contains transformers
IEstimator<ITransformer> pipeline = BuildPipeline(mlContext);
// Create ML.NET model from pipeline
ITransformer mlModel = pipeline.Fit(inputDataView);
// Save model
SaveModel(mlContext, mlModel, MLNET_MODEL, inputDataView.Schema);
}
public static IEstimator<ITransformer> BuildPipeline(MLContext mlContext)
{
");
if (PreTrainerTransforms.Count > 0)
{
this.Write(" // Data process configuration with pipeline data transformations to:\r" +
"\n // 1. Score using provided onnx model\r\n // 2. Map scores" +
" to labels to make model output easier to understand and use\r\n var pi" +
"peline = ");
for (int i = 0; i < PreTrainerTransforms.Count; i++)
{
if (i > 0)
{
Write("\r\n .Append(");
}
Write("mlContext.Transforms." + PreTrainerTransforms[i]);
if (i > 0)
{
Write(")");
}
}
this.Write(";\r\n");
}
this.Write(@" return pipeline;
}
private static void SaveModel(MLContext mlContext, ITransformer mlModel, string modelRelativePath, DataViewSchema modelInputSchema)
{
// Save/persist the trained model to a .ZIP file
Console.WriteLine($""=============== Saving the model ==============="");
mlContext.Model.Save(mlModel, modelInputSchema, GetAbsolutePath(modelRelativePath));
Console.WriteLine(""The model is saved to {0}"", GetAbsolutePath(modelRelativePath));
}
public static string GetAbsolutePath(string relativePath)
{
FileInfo _dataRoot = new FileInfo(typeof(Program).Assembly.Location);
string assemblyFolderPath = _dataRoot.Directory.FullName;
string fullPath = Path.Combine(assemblyFolderPath, relativePath);
return fullPath;
}
}
}
");
return(this.GenerationEnvironment.ToString());
}