private static byte[] GetSentimentModel(SentimentList value)
{
string modelName = "";
switch (value)
{
case SentimentList.Common:
{
modelName = "Model_ML_Common.zip";
break;
}
case SentimentList.Movie:
{
modelName = "Model_ML_Movie.zip";
break;
}
case SentimentList.Shop:
{
modelName = "Model_ML_Shop.zip";
break;
}
}
return(File.ReadAllBytes(modelName));
}
private static void FindPath(SentimentList value)
{
string dataset = "";
switch (value)
{
case SentimentList.Common:
dataset = Resource.Dataset_ML_Common;
break;
case SentimentList.Movie:
dataset = Resource.Dataset_ML_Movie;
break;
case SentimentList.Shop:
dataset = Resource.Dataset_ML_Shop;
break;
}
var domain = AppDomain.CurrentDomain.BaseDirectory.Split(Path.DirectorySeparatorChar);
StringBuilder path = new StringBuilder();
for (int i = 0; i < domain.Length - 4; i++)
{
path.Append($"{domain[i]}/");
}
DataListPath = path.Append($@"DataSet/{dataset}").ToString();
}
private static string FindPath(SentimentList value)
{
string modelName = "";
switch (value)
{
case SentimentList.Common: modelName = Resource.Model_ML_Name_Common;
break;
case SentimentList.Movie: modelName = Resource.Model_ML_Name_Movie;
break;
case SentimentList.Shop: modelName = Resource.Model_ML_Name_Shop;
break;
}
var domain = AppDomain.CurrentDomain.BaseDirectory.Split(Path.DirectorySeparatorChar);
StringBuilder path = new StringBuilder();
for (int i = 0; i < domain.Length - 4; i++)
{
path.Append($"{domain[i]}/");
}
path.Append($@"Models/{modelName}");
return(path.ToString());
}
/// <summary>
/// Get the split Dataset as Trainset and Testset, default Test Fraction is 0.2(20% from the entire dataset)
/// </summary>
/// <param name="mLContext"></param>
/// <param name="testFraction"></param>
/// <returns></returns>
public static TrainTestData GetTrainTestDataset(MLContext mLContext, SentimentList value, double testFraction = 0.2)
{
FindPath(value);
var dataView = GetDataset(mLContext);
TrainTestData trainTest = mLContext.Data.TrainTestSplit(dataView, testFraction: testFraction);
return(trainTest);
}
/// <summary>
/// Pass the string that wanted to be evaluated, returns false for toxic text and true for non-toxic text.
/// </summary>
/// <param name="sentiment"></param>
/// <param name="value"></param>
/// <returns></returns>
public static ModelOutput GetSentiment(string sentiment, SentimentList value)
{
MLContext mlContext = new MLContext();
Stream modelStream = new MemoryStream(GetSentimentModel(value));
// Load the model
ITransformer mlModel = mlContext.Model.Load(modelStream, out DataViewSchema inputSchema);
var predEngine = mlContext.Model.CreatePredictionEngine <ModelInput, ModelOutput>(mlModel);
// Try a single prediction
ModelOutput predictionResult = predEngine.Predict(new ModelInput {
SentimentText = sentiment
});
return(predictionResult);
}
public static void CreateModel(SentimentList value)
{
//Get the dataset
TrainTestData dataSet = DataSet.GetTrainTestDataset(_mLContext, value, 0.1);
//Get the model training pipeline
var pipeline = ModelTrainer.BuildTrainingPipeline(_mLContext, value);
//Get the trained model
var model = ModelTrainer.TrainModel(dataSet.TrainSet, pipeline);
//Evaluate the Model
ModelEvaluator.Evaluate(_mLContext, model, dataSet.TestSet);
//Save the Model
SaveModel(_mLContext, model, dataSet.TrainSet.Schema, value);
}
public static void GetListOfSentiments()
{
sentiments = SentimentList.getsentiments();
}
public static void SaveModel(MLContext mLContext, ITransformer model, DataViewSchema modelInputSchema, SentimentList value)
{
mLContext.Model.Save(model, modelInputSchema, FindPath(value));
Console.WriteLine("<========================================================>");
Console.WriteLine("<=== Model Saved to the current application directory ===>");
}
public static IEstimator <ITransformer> BuildTrainingPipeline(MLContext mLContext, SentimentList value)
{
var dataProcessPipeline = mLContext.Transforms.Text.FeaturizeText(outputColumnName: "Features", inputColumnName: nameof(ModelInput.SentimentText));
dynamic trainingPipeline = null;
switch (value)
{
case SentimentList.Common:
{
var trainer = mLContext.BinaryClassification.Trainers.SgdCalibrated(new SgdCalibratedTrainer.Options()
{
LearningRate = 0.5f,
L2Regularization = 1E-06f,
ConvergenceTolerance = 0.0001f,
NumberOfIterations = 100,
Shuffle = true,
LabelColumnName = "Label",
FeatureColumnName = "Features"
});
trainingPipeline = dataProcessPipeline.Append(trainer);
}
break;
case SentimentList.Movie:
{
var trainer = mLContext.BinaryClassification.Trainers.SgdCalibrated(new SgdCalibratedTrainer.Options()
{
LearningRate = 0.5f,
L2Regularization = 1E-06f,
ConvergenceTolerance = 0.0001f,
NumberOfIterations = 50,
Shuffle = true,
LabelColumnName = "Label",
FeatureColumnName = "Features"
});
trainingPipeline = dataProcessPipeline.Append(trainer);
}
break;
case SentimentList.Shop:
{
var trainer = mLContext.BinaryClassification.Trainers.SdcaLogisticRegression(new SdcaLogisticRegressionBinaryTrainer.Options()
{
BiasLearningRate = 0.5f,
L2Regularization = 1E-06f,
ConvergenceTolerance = 0.0001f,
MaximumNumberOfIterations = 100,
Shuffle = true,
LabelColumnName = "Label",
FeatureColumnName = "Features"
});
trainingPipeline = dataProcessPipeline.Append(trainer);
}
break;
}
return(trainingPipeline);
}
