public async Task OnPostAsync()
{
Message = "LOADING...";
string assetsRelativePath = @"../../../assets";
string assetsPath = GetAbsolutePath(assetsRelativePath);
var imagesFolder = Path.Combine(assetsPath, "inputs", "images-for-predictions", "tmp-web-image");
var imageClassifierZip = Path.Combine(assetsPath, "inputs", "MLNETModel", "imageClassifier.zip");
imagesFolder += @"/" + ImageFile.FileName;
try
{
using (var stream = System.IO.File.Create(imagesFolder))
{
await ImageFile.CopyToAsync(stream);
}
var modelScorer = new ModelScorer(imagesFolder, imageClassifierZip);
Predict = modelScorer.ClassifySingleImage();
}
catch (Exception ex)
{
Message = ex.Message;
}
Message = "...";
//return RedirectToPage("./Index");
}
private static void Main(string[] args)
{
//Create the MLContext to share across components for deterministic results
MLContext mlContext = new MLContext(seed: 1); //Seed set to any number so you have a deterministic environment
//STEP 1: Common data loading
DataLoader dataLoader = new DataLoader(mlContext);
var fullData = dataLoader.GetDataView(DataPath);
(IDataView trainingDataView, IDataView testingDataView) = mlContext.Clustering.TrainTestSplit(fullData, testFraction: 0.2);
//STEP 2: Process data transformations in pipeline
var dataProcessor = new DataProcessor(mlContext);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (Optional) Peek data in training DataView after applying the ProcessPipeline's transformations
Common.ConsoleHelper.PeekDataViewInConsole <IrisData>(mlContext, trainingDataView, dataProcessPipeline, 10);
Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 10);
// STEP 3: Create and train the model
var modelBuilder = new ModelBuilder <IrisData, IrisPrediction>(mlContext, dataProcessPipeline);
var trainer = mlContext.Clustering.Trainers.KMeans(features: "Features", clustersCount: 3);
modelBuilder.AddTrainer(trainer);
var trainedModel = modelBuilder.Train(trainingDataView);
// STEP4: Evaluate accuracy of the model
var metrics = modelBuilder.EvaluateClusteringModel(testingDataView);
Common.ConsoleHelper.PrintClusteringMetrics(trainer.ToString(), metrics);
// STEP5: Save/persist the model as a .ZIP file
modelBuilder.SaveModelAsFile(ModelPath);
Console.WriteLine("=============== End of training process ===============");
Console.WriteLine("=============== Predict a cluster for a single case (Single Iris data sample) ===============");
// Test with one sample text
var sampleIrisData = new IrisData()
{
SepalLength = 3.3f,
SepalWidth = 1.6f,
PetalLength = 0.2f,
PetalWidth = 5.1f,
};
//Create the clusters: Create data files and plot a chart
var modelScorer = new ModelScorer <IrisData, IrisPrediction>(mlContext);
modelScorer.LoadModelFromZipFile(ModelPath);
var prediction = modelScorer.PredictSingle(sampleIrisData);
Console.WriteLine($"Cluster assigned for setosa flowers:" + prediction.SelectedClusterId);
Console.WriteLine("=============== End of process, hit any key to finish ===============");
Console.ReadKey();
}
public static void BuildAndTrainModel(string DataSetLocation, string ModelPath)
{
// Create MLContext to be shared across the model creation workflow objects
// Set a random seed for repeatable/deterministic results across multiple trainings.
var mlContext = new MLContext(seed: 0);
// STEP 1: Common data loading configuration
DataLoader dataLoader = new DataLoader(mlContext);
var trainingDataView = dataLoader.GetDataView(DataSetLocation);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessor = new DataProcessor(mlContext);
var dataProcessPipeline = dataProcessor.DataProcessPipeline;
// (OPTIONAL) Peek data (such as 2 records) in training DataView after applying the ProcessPipeline's transformations into "Features"
Common.ConsoleHelper.PeekDataViewInConsole <GitHubIssue>(mlContext, trainingDataView, dataProcessPipeline, 2);
//Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 2);
// STEP 3: Set the selected training algorithm into the modelBuilder
var modelBuilder = new Common.ModelBuilder <GitHubIssue, GitHubIssuePrediction>(mlContext, dataProcessPipeline);
var trainer = mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent("Label", "Features");
modelBuilder.AddTrainer(trainer);
modelBuilder.AddEstimator(new KeyToValueEstimator(mlContext, "PredictedLabel"));
// STEP 4: Cross-Validate with single dataset (since we don't have two datasets, one for training and for evaluate)
// in order to evaluate and get the model's accuracy metrics
Console.WriteLine("=============== Cross-validating to get model's accuracy metrics ===============");
var crossValResults = modelBuilder.CrossValidateAndEvaluateMulticlassClassificationModel(trainingDataView, 6, "Label");
ConsoleHelper.PrintMulticlassClassificationFoldsAverageMetrics("SdcaMultiClassTrainer", crossValResults);
// STEP 5: Train the model fitting to the DataSet
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
// (OPTIONAL) Try/test a single prediction by loding the model from the file, first.
GitHubIssue issue = new GitHubIssue()
{
ID = "Any-ID", Title = "Entity Framework crashes", Description = "When connecting to the database, EF is crashing"
};
var modelScorer = new ModelScorer <GitHubIssue, GitHubIssuePrediction>(mlContext);
modelScorer.LoadModelFromZipFile(ModelPath);
var prediction = modelScorer.PredictSingle(issue);
Console.WriteLine($"=============== Single Prediction - Result: {prediction.Area} ===============");
//
Common.ConsoleHelper.ConsoleWriteHeader("Training process finalized");
}
public static void VisualizeSomePredictions(MLContext mlContext,
string modelName,
string testDataLocation,
ModelScorer <DemandObservation, DemandPrediction> modelScorer,
int numberOfPredictions)
{
//Make a few prediction tests
// Make the provided number of predictions and compare with observed data from the test dataset
var testData = ReadSampleDataFromCsvFile(testDataLocation, numberOfPredictions);
for (int i = 0; i < numberOfPredictions; i++)
{
var prediction = modelScorer.PredictSingle(testData[i]);
Common.ConsoleHelper.PrintRegressionPredictionVersusObserved(prediction.PredictedCount.ToString(),
testData[i].Count.ToString());
}
}
static async Task Main(string[] args)
{
var assetsPath = ModelHelpers.GetAssetsPath(@"..\..\..\assets");
var tagsTsv = Path.Combine(assetsPath, "inputs", "data", "tags.tsv");
var imagesFolder = Path.Combine(assetsPath, "inputs", "data");
var imageClassifierZip = Path.Combine(assetsPath, "inputs", "imageClassifier.zip");
try
{
var modelScorer = new ModelScorer(tagsTsv, imagesFolder, imageClassifierZip);
modelScorer.ClassifyImages();
}
catch (Exception ex)
{
ConsoleWriteException(ex.Message);
}
ConsolePressAnyKey();
}
static void Main(string[] args)
{
string assetsRelativePath = @"..\..\..\assets";
string assetsPath = GetDataSetAbsolutePath(assetsRelativePath);
var tagsTsv = Path.Combine(assetsPath, "inputs", "data", "tags.tsv");
var imagesFolder = Path.Combine(assetsPath, "inputs", "data");
var imageClassifierZip = Path.Combine(assetsPath, "inputs", "imageClassifier.zip");
try
{
var modelScorer = new ModelScorer(tagsTsv, imagesFolder, imageClassifierZip);
modelScorer.ClassifyImages();
}
catch (Exception ex)
{
ConsoleWriteException(ex.Message);
}
ConsolePressAnyKey();
}
public static void Run()
{
string assetsRelativePath = @"../../../assets";
string assetsPath = GetAbsolutePath(assetsRelativePath);
var tagsTsv = Path.Combine(assetsPath, "inputs", "data", "images_list.tsv");
var imagesFolder = Path.Combine(assetsPath, "inputs", "data");
var imageClassifierZip = Path.Combine(assetsPath, "inputs", "imageClassifier.zip");
try
{
var modelScorer = new ModelScorer(tagsTsv, imagesFolder, imageClassifierZip);
modelScorer.ClassifyImages();
}
catch (Exception ex)
{
ConsoleHelpers.ConsoleWriteException(ex.Message);
}
ConsoleHelpers.ConsolePressAnyKey();
}
static void Main(string[] args)
{
string assetsRelativePath = @"../../../assets";
string assetsPath = GetAbsolutePath(assetsRelativePath);
var imagesFolder = Path.Combine(assetsPath, "inputs", "images-for-predictions");
var imageClassifierZip = Path.Combine(assetsPath, "inputs", "MLNETModel", "imageClassifier.zip");
try
{
var modelScorer = new ModelScorer(imagesFolder, imageClassifierZip);
modelScorer.ClassifyImages();
}
catch (Exception ex)
{
ConsoleWriteException(ex.ToString());
}
ConsolePressAnyKey();
}
public Labeler(string modelPath, string repoOwner = "", string repoName = "", string accessToken = "")
{
_modelPath = modelPath;
_repoOwner = repoOwner;
_repoName = repoName;
_mlContext = new MLContext(seed: 1);
//Load file model into ModelScorer
_modelScorer = new ModelScorer <GitHubIssue, GitHubIssuePrediction>(_mlContext);
_modelScorer.LoadModelFromZipFile(_modelPath);
//Configure Client to access a GitHub repo
if (accessToken != string.Empty)
{
var productInformation = new ProductHeaderValue("MLGitHubLabeler");
_client = new GitHubClient(productInformation)
{
Credentials = new Credentials(accessToken)
};
}
}
static async Task Main(string[] args)
{
// Running inside Visual Studio, $SolutionDir/assets is automatically passed as argument
// If you execute from the console, pass as argument the location of the assets folder
// Otherwise, it will search for assets in the executable's folder
var assetsPath = args.Length > 0 ? args[0] : ModelHelpers.GetAssetsPath();
var tagsTsv = Path.Combine(assetsPath, "inputs", "data", "tags.tsv");
var imagesFolder = Path.Combine(assetsPath, "inputs", "data");
var inceptionPb = Path.Combine(assetsPath, "inputs", "inception", "tensorflow_inception_graph.pb");
var labelsTxt = Path.Combine(assetsPath, "inputs", "inception", "imagenet_comp_graph_label_strings.txt");
try
{
var modelEvaluator = new ModelScorer(tagsTsv, imagesFolder, inceptionPb, labelsTxt);
modelEvaluator.Score();
}
catch (Exception ex)
{
Console.WriteLine($"Exception: {ex.Message}");
}
Console.ReadKey();
}
public PersonController(PersonDbContext context)
{
_context = context;
PresentsHelper.ExtractAllPresents(_context.Persons);
_modelScorer = new ModelScorer("model.zip");
}
public static void BuildAndTrainModel(string DataSetLocation, string ModelPath, MyTrainerStrategy selectedStrategy)
{
// Create MLContext to be shared across the model creation workflow objects
// Set a random seed for repeatable/deterministic results across multiple trainings.
var mlContext = new MLContext(seed: 0);
// STEP 1: Common data loading configuration
var textLoader = GitHubLabelerTextLoaderFactory.CreateTextLoader(mlContext);
var trainingDataView = textLoader.Read(DataSetLocation);
// STEP 2: Common data process configuration with pipeline data transformations
var dataProcessPipeline = GitHubLabelerDataProcessPipelineFactory.CreateDataProcessPipeline(mlContext);
// (OPTIONAL) Peek data (such as 2 records) in training DataView after applying the ProcessPipeline's transformations into "Features"
Common.ConsoleHelper.PeekDataViewInConsole <GitHubIssue>(mlContext, trainingDataView, dataProcessPipeline, 2);
//Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, "Features", trainingDataView, dataProcessPipeline, 2);
// STEP 3: Create the selected training algorithm/trainer
IEstimator <ITransformer> trainer = null;
switch (selectedStrategy)
{
case MyTrainerStrategy.SdcaMultiClassTrainer:
trainer = mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent(DefaultColumnNames.Label,
DefaultColumnNames.Features);
break;
case MyTrainerStrategy.OVAAveragedPerceptronTrainer:
{
// Create a binary classification trainer.
var averagedPerceptronBinaryTrainer = mlContext.BinaryClassification.Trainers.AveragedPerceptron(DefaultColumnNames.Label,
DefaultColumnNames.Features,
numIterations: 10);
// Compose an OVA (One-Versus-All) trainer with the BinaryTrainer.
// In this strategy, a binary classification algorithm is used to train one classifier for each class, "
// which distinguishes that class from all other classes. Prediction is then performed by running these binary classifiers, "
// and choosing the prediction with the highest confidence score.
trainer = new Ova(mlContext, averagedPerceptronBinaryTrainer);
break;
}
default:
break;
}
//Set the trainer/algorithm
var modelBuilder = new Common.ModelBuilder <GitHubIssue, GitHubIssuePrediction>(mlContext, dataProcessPipeline);
modelBuilder.AddTrainer(trainer);
modelBuilder.AddEstimator(mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel"));
// STEP 4: Cross-Validate with single dataset (since we don't have two datasets, one for training and for evaluate)
// in order to evaluate and get the model's accuracy metrics
Console.WriteLine("=============== Cross-validating to get model's accuracy metrics ===============");
var crossValResults = modelBuilder.CrossValidateAndEvaluateMulticlassClassificationModel(trainingDataView, 6, "Label");
ConsoleHelper.PrintMulticlassClassificationFoldsAverageMetrics(trainer.ToString(), crossValResults);
// STEP 5: Train the model fitting to the DataSet
Console.WriteLine("=============== Training the model ===============");
modelBuilder.Train(trainingDataView);
// (OPTIONAL) Try/test a single prediction with the "just-trained model" (Before saving the model)
GitHubIssue issue = new GitHubIssue()
{
ID = "Any-ID", Title = "WebSockets communication is slow in my machine", Description = "The WebSockets communication used under the covers by SignalR looks like is going slow in my development machine.."
};
var modelScorer = new ModelScorer <GitHubIssue, GitHubIssuePrediction>(mlContext, modelBuilder.TrainedModel);
var prediction = modelScorer.PredictSingle(issue);
Console.WriteLine($"=============== Single Prediction just-trained-model - Result: {prediction.Area} ===============");
//
// STEP 6: Save/persist the trained model to a .ZIP file
Console.WriteLine("=============== Saving the model to a file ===============");
modelBuilder.SaveModelAsFile(ModelPath);
Common.ConsoleHelper.ConsoleWriteHeader("Training process finalized");
}
static void Main(string[] args)
{
try
{
ArgCollection argCollection = ArgCollection.GetInstance(args);
if (argCollection.ExtractOptionalFlag("help"))
{
Console.WriteLine("");
Console.WriteLine(UsageMessage);
Console.WriteLine(HelpMessage);
return;
}
string optimizerName = argCollection.ExtractOptional <string>("optimizer", "BrentThenGrid");
string keepTestName = argCollection.ExtractOptional <string>("keepTest", "AlwaysKeep");
string skipRowIndexFileNameOrNull = argCollection.ExtractOptional <string>("skipRowIndexFile", null);
argCollection.CheckNoMoreOptions();
string treeFileName = argCollection.ExtractNext <string>("treeFile");
string predictorFileName = argCollection.ExtractNext <string>("predictorFile");
string targetFileName = argCollection.ExtractNext <string>("targetFile");
string leafDistributionName = argCollection.ExtractNext <string>("leafDistribution");
string nullDataGeneratorName = argCollection.ExtractNext <string>("nullDataGenerator");
string niceName = argCollection.ExtractNext <string>("niceName");
string outputDirectory = argCollection.ExtractNext <string>("outputDirectory");
RangeCollection pieceIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("pieceIndexRange");
int pieceCount = argCollection.ExtractNext <int>("pieceCount");
RangeCollection nullIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("nullIndexRange");
argCollection.CheckThatEmpty();
if (!PhyloDDriver.ValidateDistribution(leafDistributionName))
{
Console.WriteLine("{0} is not a recognized distribution name. Please choose a name from the following list:", leafDistributionName);
foreach (string name in PhyloDDriver.GetDistributionNames())
{
Console.WriteLine("\t{0}", name);
}
throw new ArgumentException("Invalid distribution name.");
}
RangeCollection skipRowIndexRangeCollectionOrNull = (null == skipRowIndexFileNameOrNull) || skipRowIndexFileNameOrNull == "null" ? null : RangeCollection.Parse(File.ReadAllText(skipRowIndexFileNameOrNull));
KeepTest <Dictionary <string, string> > keepTest = KeepTest <Dictionary <string, string> > .GetInstance(null, keepTestName);
SpecialFunctions.CheckCondition(pieceIndexRangeCollection.IsBetween(0, pieceCount - 1), "pieceIndex must be at least 0 and less than pieceCount");
SpecialFunctions.CheckCondition(nullIndexRangeCollection.IsBetween(-1, int.MaxValue), "nullIndex must be at least -1");
PhyloTree aPhyloTree = PhyloTree.GetInstance(treeFileName, null);
ModelScorer modelScorer = ModelScorer.GetInstance(aPhyloTree, leafDistributionName, optimizerName);
ModelEvaluator modelEvaluator = ModelEvaluator.GetInstance(leafDistributionName, modelScorer);
PhyloDDriver driver = PhyloDDriver.GetInstance();
driver.Run(
modelEvaluator,
predictorFileName, targetFileName,
leafDistributionName, nullDataGeneratorName,
keepTest, skipRowIndexRangeCollectionOrNull,
niceName,
outputDirectory,
pieceIndexRangeCollection, pieceCount,
nullIndexRangeCollection,
optimizerName);
//Console.Write("Press enter to exist.");
//Console.Read();
}
catch (Exception exception)
{
Console.WriteLine("");
Console.WriteLine(exception.Message);
if (exception.InnerException != null)
{
Console.WriteLine(exception.InnerException.Message);
}
Console.WriteLine("");
Console.WriteLine(UsageMessage);
throw;
}
}
/// <summary>
/// Does the work.
/// </summary>
public override void DoWork()
{
// get our input data and null the field to make sure we don't serialize it back
InputData inputData = mInputData;
mInputData = null;
// get the job-specific names of input files
FileDefCollection fileDefs = Job.FileDefs;
string treeFileName = Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).LocalName;
string predictorFileName = Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).LocalName;
string targetFileName = Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).LocalName;
string skipRowIndexFileName = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName).LocalName;
// construct RangeCollections
RangeCollection pieceIndexRangeCollection = RangeCollection.Parse(inputData.PieceIndexRange);
RangeCollection nullIndexRangeCollection = RangeCollection.Parse(inputData.NullIndexRange);
RangeCollection skipRowIndexRangeCollection;
FileInfo fileInfo = new FileInfo(skipRowIndexFileName);
if (fileInfo.Length > 0)
{
skipRowIndexRangeCollection = RangeCollection.Parse(File.ReadAllText(skipRowIndexFileName));
}
else
{
skipRowIndexRangeCollection = null;
}
// do the rest
PhyloTree aPhyloTree = PhyloTree.GetInstance(treeFileName, null);
ModelScorer modelScorer =
ModelScorer.GetInstance(aPhyloTree, inputData.LeafDistributionName, inputData.OptimizerName);
ModelEvaluator modelEvaluator = ModelEvaluator.GetInstance(inputData.LeafDistributionName, modelScorer);
KeepTest <Dictionary <string, string> > keepTest =
KeepTest <Dictionary <string, string> > .GetInstance(null, inputData.KeepTestName);
PhyloDDriver driver = PhyloDDriver.GetInstance();
// create a name for the temporary job sandbox. This directory gets created by driver.Run(...)
string agentOutputDirectoryName =
Path.Combine(Environment.CurrentDirectory, String.Format(CultureInfo.InvariantCulture, "{0}.{1}", Job.JobId, Task.TaskId));
// save the standard out and standard error in memory streams
using (MemoryStream streamOut = new MemoryStream(), streamError = new MemoryStream())
{
try
{
// redirect the outputs
using (
StreamWriter writerOut = new StreamWriter(streamOut),
writerError = new StreamWriter(streamError))
{
Console.SetOut(writerOut);
Console.SetError(writerError);
try
{
// run the model
string outputFileName = driver.Run(
modelEvaluator,
predictorFileName, targetFileName,
inputData.LeafDistributionName, inputData.NullDataGeneratorName,
keepTest, skipRowIndexRangeCollection,
inputData.NiceName,
agentOutputDirectoryName,
pieceIndexRangeCollection, inputData.PieceCount,
nullIndexRangeCollection,
inputData.OptimizerName);
// this is the expected output file name -- save this so it can be written on the master side with the same name.
mOutputFileName = Path.GetFileName(outputFileName);
mLocalOutputFileName = Path.Combine(inputData.LocalOutputDirectoryName, mOutputFileName);
// get the output data
string fullOutputPath = Path.Combine(agentOutputDirectoryName, mOutputFileName);
if (!File.Exists(fullOutputPath))
{
TaskResult.FailureReason = TaskFailureReason.MissingOutput;
TaskResult.FailureMessage = String.Format(CultureInfo.CurrentCulture, "Cannot find output file '{0}'", targetFileName);
TaskResult.Status = TaskAssignmentStatus.Failed;
}
using (StreamReader outputData = new StreamReader(fullOutputPath))
{
mOutputData = outputData.ReadToEnd();
}
}
finally
{
// this finally is to make sure we delete the folder
// get rid of the sandbox
Directory.Delete(agentOutputDirectoryName, true);
}
}
}
finally
{
// this finally is to make sure we get console output
Encoding encoding = Encoding.Default;
TaskResult.StandardOutput = encoding.GetString(streamOut.GetBuffer());
TaskResult.StandardError = encoding.GetString(streamError.GetBuffer());
}
}
}
