private void Init(PipeModel meta)
{
if (_classifier == null)
{
meta.Model = "intent.model";
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model),
ModelDir = Settings.ModelDir,
ModelName = meta.Model,
Word2VecFilePath = Configuration.GetValue <string>("wordvecModel")
};
if (!String.IsNullOrEmpty(options.Word2VecFilePath))
{
string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
options.Word2VecFilePath = options.Word2VecFilePath.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar);
}
_classifier = new ClassifierFactory <SentenceFeatureExtractor>(options, SupportedLanguage.English);
string classifierName = Configuration.GetValue <String>($"classifer");
_classifier.GetClassifer(classifierName);
}
}
public void SpookyAuthorIdentification()
{
var reader = new KaggleTextDataReader();
var sentences = reader.Read(new ReaderOptions {
FileName = "train.csv"
});
var tokenizer = new TokenizerFactory(new TokenizationOptions {
}, SupportedLanguage.English);
tokenizer.GetTokenizer <TreebankTokenizer>();
var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
for (int i = 0; i < newSentences.Count; i++)
{
newSentences[i].Id = sentences[i].Id;
newSentences[i].Label = sentences[i].Label;
}
sentences = newSentences.ToList();
sentences.Shuffle();
var dataset = sentences.Take(2000).ToList().Split(0.7M);
var options = new ClassifyOptions
{
ModelDir = AppContext.BaseDirectory,
ModelFilePath = Path.Combine(AppContext.BaseDirectory, "nb.model"),
Dimension = 300
};
var classifier = new ClassifierFactory <SentenceFeatureExtractor>(options, SupportedLanguage.English);
classifier.GetClassifer("NaiveBayesClassifier");
classifier.Train(dataset.Item1);
int correct = 0;
int total = 0;
dataset.Item2.ForEach(td =>
{
var classes = classifier.Classify(td);
if (td.Label == classes[0].Item1)
{
correct++;
}
total++;
});
var accuracy = (float)correct / total;
Assert.IsTrue(accuracy > 0.5);
}
private void Init(PipeModel meta)
{
if (_classifier == null)
{
meta.Model = "intent.model";
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model),
ModelDir = Settings.ModelDir,
ModelName = meta.Model
};
_classifier = new ClassifierFactory <SentenceFeatureExtractor>(options, SupportedLanguage.English);
string classifierName = Configuration.GetValue <String>($"classifer");
_classifier.GetClassifer(classifierName);
}
}
