Ejemplo n.º 1
0
        private static void Main(string[] args)
        {
            //Create the MLContext to share across components for deterministic results
            MLContext mlContext = new MLContext(seed: 1);  //Seed set to any number so you have a deterministic environment

            // STEP 1: Common data loading configuration
            IDataView fullData = mlContext.Data.ReadFromTextFile(path: DataPath,
                                                                 columns: new[]
            {
                new TextLoader.Column(DefaultColumnNames.Label, DataKind.R4, 0),
                new TextLoader.Column(nameof(IrisData.SepalLength), DataKind.R4, 1),
                new TextLoader.Column(nameof(IrisData.SepalWidth), DataKind.R4, 2),
                new TextLoader.Column(nameof(IrisData.PetalLength), DataKind.R4, 3),
                new TextLoader.Column(nameof(IrisData.PetalWidth), DataKind.R4, 4),
            },
                                                                 hasHeader: true,
                                                                 separatorChar: '\t');

            //Split dataset in two parts: TrainingDataset (80%) and TestDataset (20%)
            (IDataView trainingDataView, IDataView testingDataView) = mlContext.Clustering.TrainTestSplit(fullData, testFraction: 0.2);

            //STEP 2: Process data transformations in pipeline
            var dataProcessPipeline = mlContext.Transforms.Concatenate(DefaultColumnNames.Features, nameof(IrisData.SepalLength), nameof(IrisData.SepalWidth), nameof(IrisData.PetalLength), nameof(IrisData.PetalWidth));

            // (Optional) Peek data in training DataView after applying the ProcessPipeline's transformations
            Common.ConsoleHelper.PeekDataViewInConsole(mlContext, trainingDataView, dataProcessPipeline, 10);
            Common.ConsoleHelper.PeekVectorColumnDataInConsole(mlContext, DefaultColumnNames.Features, trainingDataView, dataProcessPipeline, 10);

            // STEP 3: Create and train the model
            var trainer          = mlContext.Clustering.Trainers.KMeans(featureColumn: DefaultColumnNames.Features, clustersCount: 3);
            var trainingPipeline = dataProcessPipeline.Append(trainer);
            var trainedModel     = trainingPipeline.Fit(trainingDataView);

            // STEP4: Evaluate accuracy of the model
            IDataView predictions = trainedModel.Transform(testingDataView);
            var       metrics     = mlContext.Clustering.Evaluate(predictions, score: DefaultColumnNames.Score, features: DefaultColumnNames.Features);

            ConsoleHelper.PrintClusteringMetrics(trainer.ToString(), metrics);

            // STEP5: Save/persist the model as a .ZIP file
            using (var fs = new FileStream(ModelPath, FileMode.Create, FileAccess.Write, FileShare.Write))
                mlContext.Model.Save(trainedModel, fs);

            Console.WriteLine("=============== End of training process ===============");

            Console.WriteLine("=============== Predict a cluster for a single case (Single Iris data sample) ===============");

            // Test with one sample text
            var sampleIrisData = new IrisData()
            {
                SepalLength = 3.3f,
                SepalWidth  = 1.6f,
                PetalLength = 0.2f,
                PetalWidth  = 5.1f,
            };

            using (var stream = new FileStream(ModelPath, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                ITransformer model = mlContext.Model.Load(stream);
                // Create prediction engine related to the loaded trained model
                var predEngine = model.CreatePredictionEngine <IrisData, IrisPrediction>(mlContext);

                //Score
                var resultprediction = predEngine.Predict(sampleIrisData);

                Console.WriteLine($"Cluster assigned for setosa flowers:" + resultprediction.SelectedClusterId);
            }

            Console.WriteLine("=============== End of process, hit any key to finish ===============");
            Console.ReadKey();
        }
Ejemplo n.º 2
0
        // This method gets called by the runtime. Use this method to add services to the container.
        public void ConfigureServices(IServiceCollection services)
        {
            services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2);

            // Register Types in IoC container for DI

            //MLContext created as singleton for the whole ASP.NET Core app
            services.AddSingleton <MLContext>((ctx) =>
            {
                //Seed set to any number so you have a deterministic environment
                return(new MLContext(seed: 1));
            });

            //ML Model (ITransformed) created as singleton for the whole ASP.NET Core app. Loads from .zip file here.
            services.AddSingleton <ITransformer,
                                   TransformerChain <ITransformer> > ((ctx) =>
            {
                MLContext mlContext      = ctx.GetRequiredService <MLContext>();
                string modelFilePathName = Configuration["MLModel:MLModelFilePath"];

                ITransformer mlModel;
                using (var fileStream = File.OpenRead(modelFilePathName))
                    mlModel = mlContext.Model.Load(fileStream);

                return((TransformerChain <ITransformer>)mlModel);
            });

            // PredictionEngine created as Transient since it is not thread safe
            // This injected PredictionEngine is ONLY used on the MLModelEngineSimple implementation
            // This injected PredictionEngine is NOT used when using the Object Pooling or ThreadStatic approaches
            services.AddTransient <PredictionEngine <SampleObservation, SamplePrediction> >((ctx) =>
            {
                MLContext mlContext  = ctx.GetRequiredService <MLContext>();
                ITransformer mlmodel = ctx.GetRequiredService <ITransformer>();
                var predEngine       = mlmodel.CreatePredictionEngine <SampleObservation, SamplePrediction>(mlContext);
                return(predEngine);
            });

            // OPTION A:
            // Using MLModelEngine ObjPooling implementation
            //
            services.AddSingleton <IMLModelEngine <SampleObservation, SamplePrediction>,
                                   MLModelEngineObjPooling <SampleObservation, SamplePrediction> >();


            // OPTION B:
            // Using MLModelEngine ThreadStatic implementation
            //
            //services.AddSingleton<IMLModelEngine<SampleObservation, SamplePrediction>,
            //          MLModelEngineThreadStatic<SampleObservation, SamplePrediction>>();


            // OPTION C:
            // Using MLModelEngine Simple implementation (Create Prediction Engine for every call)
            //
            //services.AddSingleton<IMLModelEngine<SampleObservation, SamplePrediction>,
            //          MLModelEngineSimple<SampleObservation, SamplePrediction>>();



            // Using 'Factory code' when registering the engine.
            // Not needed in current implementation
            //
            //services.AddSingleton<IMLModelEngine<SampleObservation, SamplePrediction>,
            //                      MLModelEngineObjPooling<SampleObservation, SamplePrediction>>((ctx) =>
            //{
            //    MLContext mlContext = ctx.GetRequiredService<MLContext>();
            //    string modelFilePathName = Configuration["MLModelFilePath"];

            //    return new MLModelEngineObjPooling<SampleObservation, SamplePrediction>(mlContext,
            //                                                                            modelFilePathName);
            //});
        }
Ejemplo n.º 3
0
        public PredictionBaseModel Predict(ITransformer trainedModel, BaseModel model)
        {
            var engine = trainedModel.CreatePredictionEngine <BaseModel, PredictionBaseModel>(mlContext);

            return(engine.Predict(model));
        }
Ejemplo n.º 4
0
        public Task <TrainedModelPrediction> UseModelWithSingleItem(ITransformer model, Flute.Shared.Models.TrainedModel sample)
        {
            PredictionEngine <TrainedModel, TrainedModelPrediction> predictionFunction = model.CreatePredictionEngine <TrainedModel, TrainedModelPrediction>(mlContext);

            var resultprediction = predictionFunction.Predict(new TrainedModel()
            {
                Input = sample.Input, Label = Convert.ToBoolean(sample.Label)
            });

            return(Task.FromResult(resultprediction));
        }
Ejemplo n.º 5
0
        public StudentPredictionModel Predict(ITransformer trainedModel, StudentTrainingModel model)
        {
            var engine = trainedModel.CreatePredictionEngine <StudentTrainingModel, StudentPredictionModel>(mlContext);

            return(engine.Predict(model));
        }
Ejemplo n.º 6
0
        public PredictionEngine <TData, TPrediction> Create()
        {
            var predictionEngine = _model.CreatePredictionEngine <TData, TPrediction>(_mlContext);

            return(predictionEngine);
        }
Ejemplo n.º 7
0
        private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
        {
            //user model with single data time
            PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine <SentimentData, SentimentPrediction>(mlContext);
            SentimentData sampleStatement = new SentimentData
            {
                SentimentText = "This was a very bad steak"
            };
            var resultprediction = predictionFunction.Predict(sampleStatement);

            Console.WriteLine();
            Console.WriteLine("=== Prediction test of model with a single sample and test dataset ===");
            Console.WriteLine();
            Console.WriteLine($"Sentiment: {sampleStatement.SentimentText} | Prediction: {(Convert.ToBoolean(resultprediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultprediction.Probability} ");
            Console.WriteLine("=== End of predictions ===");
            Console.WriteLine();
        }
Ejemplo n.º 8
0
        //static TextLoader _textLoader; not used

        static void Main(string[] args)
        {
            // STEP 1: Common data loading configuration
            MLContext mlContext = new MLContext(seed: 0);

            IDataView baseTrainingDataView = mlContext.Data.LoadFromTextFile <TaxiTrip>(_trainDataPath, hasHeader: true, separatorChar: ','); // ReadFromTextFile is deprecated
            IDataView testDataView         = mlContext.Data.LoadFromTextFile <TaxiTrip>(_testDataPath, hasHeader: true, separatorChar: ',');

            //Sample code of removing extreme data like "outliers" for FareAmounts higher than $150 and lower than $1 which can be error-data
            var       cnt = baseTrainingDataView.GetColumn <float>(mlContext, nameof(TaxiTrip.FareAmount)).Count();
            IDataView trainingDataView = mlContext.Data.FilterRowsByColumn(baseTrainingDataView, nameof(TaxiTrip.FareAmount), lowerBound: 1, upperBound: 150);


            // STEP 2: Common data process configuration with pipeline data transformations
            var dataProcessPipeline = mlContext.Transforms.CopyColumns(outputColumnName: DefaultColumnNames.Label, inputColumnName: nameof(TaxiTrip.FareAmount))
                                      .Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "VendorIdEncoded", inputColumnName: nameof(TaxiTrip.VendorId)))
                                      .Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "RateCodeEncoded", inputColumnName: nameof(TaxiTrip.RateCode)))
                                      .Append(mlContext.Transforms.Categorical.OneHotEncoding(outputColumnName: "PaymentTypeEncoded", inputColumnName: nameof(TaxiTrip.PaymentType)))
                                      .Append(mlContext.Transforms.Normalize(outputColumnName: nameof(TaxiTrip.PassengerCount), mode: NormalizerMode.MeanVariance))
                                      .Append(mlContext.Transforms.Normalize(outputColumnName: nameof(TaxiTrip.TripTime), mode: NormalizerMode.MeanVariance))
                                      .Append(mlContext.Transforms.Normalize(outputColumnName: nameof(TaxiTrip.TripDistance), mode: NormalizerMode.MeanVariance))
                                      .Append(mlContext.Transforms.Concatenate(DefaultColumnNames.Features, "VendorIdEncoded", "RateCodeEncoded", "PaymentTypeEncoded", nameof(TaxiTrip.PassengerCount)
                                                                               , nameof(TaxiTrip.TripTime), nameof(TaxiTrip.TripDistance)));


            // STEP 3: Set the training algorithm, then create and config the modelBuilder - Selected Trainer (SDCA Regression algorithm)
            var trainer          = mlContext.Regression.Trainers.StochasticDualCoordinateAscent(labelColumnName: DefaultColumnNames.Label, featureColumnName: DefaultColumnNames.Features);
            var trainingPipeline = dataProcessPipeline.Append(trainer);


            //STEP 4: Train the model fitting to the DataSet
            //The pipeline is trained on the dataset that has been loaded and transformed.
            ITransformer trainedModel = trainingPipeline.Fit(trainingDataView); // changed var trainedModel to ITransformer trainedModel


            // STEP 5: Evaluate the model and show accuracy stats
            // create a copy of the testDataView in IDataView format since IDataView is immutable?
            IDataView predictions = trainedModel.Transform(testDataView);
            // assign label column (original regression value) and score column (predictive regression values) to columns in data i.e. predictions
            // apply metrics e.g. root mean squared error on data
            var metrics = mlContext.Regression.Evaluate(predictions, label: DefaultColumnNames.Label, score: DefaultColumnNames.Score);

            // print the results of the metrics
            Common.ConsoleHelper.PrintRegressionMetrics(trainer.ToString(), metrics);


            // STEP 6: Save/persist the trained model to a .ZIP file
            using (var fs = File.Create(_modelPath))
                trainedModel.SaveTo(mlContext, fs);

            Console.WriteLine("The model is saved to {0}", _modelPath);


            // STEP 7: Prediction
            //Sample:
            //vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount
            //VTS,1,1,1140,3.75,CRD,15.5

            var taxiTripSample = new TaxiTrip()
            {
                VendorId       = "VTS",
                RateCode       = "1",
                PassengerCount = 1,
                TripTime       = 1140,
                TripDistance   = 3.75f,
                PaymentType    = "CRD",
                FareAmount     = 0 // To predict. Actual/Observed = 15.5
            };


            using (var stream = new FileStream(_modelPath, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                trainedModel = mlContext.Model.Load(stream);
            }

            // Create prediction engine related to the loaded trained model
            var predEngine = trainedModel.CreatePredictionEngine <TaxiTrip, TaxiTripFarePrediction>(mlContext);

            //Score
            var resultprediction = predEngine.Predict(taxiTripSample);

            Console.WriteLine($"**********************************************************************");
            Console.WriteLine($"Predicted fare: {resultprediction.FareAmount:0.####}, actual fare: 15.5");
            Console.WriteLine($"**********************************************************************");
        }
Ejemplo n.º 9
0
        /// <summary>
        /// The main entry point of the program.
        /// </summary>
        /// <param name="args">The command line arguments.</param>
        static void Main(string[] args)
        {
            // create a machine learning context
            var context = new MLContext();

            // load the dataset in memory
            Console.WriteLine("Loading data...");
            var data = context.Data.LoadFromTextFile <ProductInfo>(
                dataPath,
                hasHeader: true,
                separatorChar: '\t');

            // split the data into 80% training and 20% testing partitions
            var partitions = context.BinaryClassification.TrainTestSplit(data, testFraction: 0.2);

            // prepare matrix factorization options
            var options = new MatrixFactorizationTrainer.Options()
            {
                MatrixColumnIndexColumnName = "ProductIDEncoded",
                MatrixRowIndexColumnName    = "CombinedProductIDEncoded",
                LabelColumnName             = "CombinedProductID",
                LossFunction = MatrixFactorizationTrainer.LossFunctionType.SquareLossOneClass,
                Alpha        = 0.01,
                Lambda       = 0.025,
            };

            // set up a training pipeline
            // step 1: map ProductID and CombinedProductID to keys
            var pipeline = context.Transforms.Conversion.MapValueToKey(
                inputColumnName: "ProductID",
                outputColumnName: "ProductIDEncoded")
                           .Append(context.Transforms.Conversion.MapValueToKey(
                                       inputColumnName: "CombinedProductID",
                                       outputColumnName: "CombinedProductIDEncoded"))

                           // step 2: find recommendations using matrix factorization
                           .Append(context.Recommendation().Trainers.MatrixFactorization(options));

            // train the model
            Console.WriteLine("Training the model...");
            ITransformer model = pipeline.Fit(partitions.TrainSet);

            Console.WriteLine();

            // evaluate the model performance
            Console.WriteLine("Evaluating the model...");
            var predictions = model.Transform(partitions.TestSet);
            var metrics     = context.Regression.Evaluate(predictions, label: "CombinedProductID", score: DefaultColumnNames.Score);

            Console.WriteLine($"  RMSE: {metrics.Rms:#.##}");
            Console.WriteLine($"  L1:   {metrics.L1:#.##}");
            Console.WriteLine($"  L2:   {metrics.L2:#.##}");
            Console.WriteLine();

            // check how well products 3 and 63 go together
            Console.WriteLine("Predicting if two products combine...");
            var engine     = model.CreatePredictionEngine <ProductInfo, ProductPrediction>(context);
            var prediction = engine.Predict(
                new ProductInfo()
            {
                ProductID         = 3,
                CombinedProductID = 63
            });

            Console.WriteLine($"  Score of products 3 and 63 combined: {prediction.Score}");
            Console.WriteLine();

            // find the top 5 combined products for product 6
            Console.WriteLine("Calculating the top 5 products for product 3...");
            var top5 = (from m in Enumerable.Range(1, 262111)
                        let p = engine.Predict(
                            new ProductInfo()
            {
                ProductID = 3,
                CombinedProductID = (uint)m
            })
                                orderby p.Score descending
                                select(ProductID: m, Score: p.Score)).Take(5);

            foreach (var t in top5)
            {
                Console.WriteLine($"  Score:{t.Score}\tProduct: {t.ProductID}");
            }

            Console.ReadLine();
        }
Ejemplo n.º 10
0
 public PredictionEngine <SentimentIssue, SentimentPrediction> CreatePredictionEngine()
 {
     return(_trainedModel.CreatePredictionEngine <SentimentIssue, SentimentPrediction>(_context));
 }
Ejemplo n.º 11
0
 public Predictor(MLContext context, ITransformer containsPeopleModel, ITransformer numberOfPeopleModel, ITransformer distanceModel)
 {
     ContainsPeoplePredictor = containsPeopleModel.CreatePredictionEngine <Data, PredictedContainsPeople>(context);
     NumberOfPeoplePredictor = numberOfPeopleModel.CreatePredictionEngine <Data, PredictedNumberOfPeople>(context);
     DistancePredictor       = distanceModel.CreatePredictionEngine <Data, PredictedDistance>(context);
 }
Ejemplo n.º 12
0
        private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
        {
            PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine <SentimentData, SentimentPrediction>(mlContext);
            // The Predict() function makes a prediction on a single row of data.
            var res = predictionFunction.Predict(new SentimentData()
            {
                SentimentText = "This was a very bad steak"
            });

            Console.WriteLine("=============== Prediction Test of model with a single sample and test dataset ===============");

            Console.WriteLine($"Prediction {res.Prediction}");


            Console.WriteLine("=============== End of Predictions ===============");
            Console.WriteLine();
        }
Ejemplo n.º 13
0
        private static void UseModelWithSingleItem(MLContext mlContext, ITransformer model)
        {
            //PredictionEngine<SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine<SentimentData, SentimentPrediction>(mlContext);
            PredictionEngine <TrabajoPlanificadoPropuestaData, TrabajoPlanificadoPrediction> predictionFunction = model.CreatePredictionEngine <TrabajoPlanificadoPropuestaData, TrabajoPlanificadoPrediction>(mlContext);
            TrabajoPlanificadoPropuestaData sampleStatement = new TrabajoPlanificadoPropuestaData();
            var resultprediction = predictionFunction.Predict(sampleStatement);

            Console.WriteLine();
            Console.WriteLine("=============== Prediction Test of model with a single sample and test dataset ===============");

            Console.WriteLine();
            //Console.WriteLine($"Sentiment: {sampleStatement.SentimentText} | Prediction: {(Convert.ToBoolean(resultprediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultprediction.Probability} ");

            Console.WriteLine("=============== End of Predictions ===============");
            Console.WriteLine();
        }
Ejemplo n.º 14
0
        public static SentimentPrediction RunModel(this MLContext mlContext, ITransformer model, string sampleText)
        {
            PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = model.CreatePredictionEngine <SentimentData, SentimentPrediction>(mlContext);

            return(predictionFunction.Predict(new SentimentData {
                SentimentText = sampleText
            }));
        }