public Span[] GetTokens(string paragraph)
{
var bin = GetFileStream("en-token.bin");
TokenizerModel model = new TokenizerModel(bin);
TokenizerME tokenizer = new TokenizerME(model);
Span[] tokens = tokenizer.tokenizePos(paragraph);
bin.close();
return(tokens);
}
public string[] Tokenization(string str)
{
Span[] tokenSpans = tokenizer.tokenizePos(str);
List <String> list = new List <String>();
foreach (Span span in tokenSpans)
{
list.Add(str.Substring(span.getStart(), span.getEnd() - span.getStart()));
}
return(list.ToArray());
}
/// <summary>
/// 使用openNLP英文斷詞。model: en-token.bin。
/// </summary>
/// <param name="sentence">需要斷詞的英文句子。</param>
/// <returns>斷完的每個Token</returns>
public string[] Tokenize(string sentence)
{
LoadTokenizer();
Span[] tokenSpans = tokenizer.tokenizePos(sentence);
List <String> list = new List <String>();
foreach (Span span in tokenSpans)
{
list.Add(sentence.Substring(span.getStart(), span.getEnd() - span.getStart()));
}
return(list.ToArray());
}
public virtual Span[] GetTerms(string sentence)
{
lock (this)
{
if (tokenizer == null)
{
Span[] span1 = new Span[1];
span1[0] = new Span(0, sentence.Length);
return(span1);
}
return(tokenizer.tokenizePos(sentence));
}
}
public virtual Span[] GetTerms(string sentence)
{
UninterruptableMonitor.Enter(this);
try
{
if (tokenizer is null)
{
Span[] span1 = new Span[1];
span1[0] = new Span(0, sentence.Length);
return(span1);
}
return(tokenizer.tokenizePos(sentence));
}
finally
{
UninterruptableMonitor.Exit(this);
}
}