public List<string> Tokenize() { var model = new TokenizerModel(File.OpenRead(BaseFolder + "en-token.bin")); var tokenizer = new TokenizerME(model); return tokenizer.Tokenize(this.Text).ToList(); }
public void TestTokenizerSimpleModel() { var model = TokenizerTestUtil.CreateMaxentTokenModel(); var tokenizer = new TokenizerME(model); var tokens = tokenizer.Tokenize("test,"); Assert.AreEqual(2, tokens.Length); Assert.AreEqual("test", tokens[0]); Assert.AreEqual(",", tokens[1]); }
public void TestCrossCompatibility() { using (var data = Tests.OpenFile("/opennlp/tools/tokenize/token.train")) { var samples = new TokenSampleStream(new PlainTextByLineStream(data)); var mlParams = new TrainingParameters(); mlParams.Set(Parameters.Iterations, "100"); mlParams.Set(Parameters.Cutoff, "0"); var model = TokenizerME.Train(samples, new TokenizerFactory("en", null, true), mlParams); var sMe = new TokenizerME(model); TokenizerMETest.TestTokenizer(sMe); var sProbs = sMe.TokenProbabilities; // --- java \/ var sFile = Path.GetTempFileName(); model.Serialize(new FileStream(sFile, FileMode.Create)); var jModel = new opennlp.tools.tokenize.TokenizerModel( OpenNLP.CreateInputStream(sFile) ); var jMe = new opennlp.tools.tokenize.TokenizerME(jModel); TestJavaTokenizer(jMe); var jProbs = jMe.getTokenProbabilities(); Assert.AreEqual(jProbs.Length, sProbs.Length); for (int i = 0; i < jProbs.Length; i++) { // one difference :( // -0.00000000000000011102230246251565 // // but still "insignificant" :) Assert.AreEqual(jProbs[i], sProbs[i], 0.0000000001d); } } }
public void TestTokenizer() { var model = TokenizerTestUtil.CreateMaxentTokenModel(); var tokenizer = new TokenizerME(model); TestTokenizer(tokenizer); }