public IList <string> TestClassicTokenizerReturnDelimiterTrue(string text)
{
var tokenizer = new ClassicTokenizer(true);
IList <string> tokens = tokenizer.Tokenize(text);
return(tokens);
}
private static void SentenceSegmentation()
{
var paragraph = "Prof. Dr. Ahmet Bey 1.6 oranında artış var dedi 2. kez. E-posta adresi [email protected] imiş! Doğru mu?";
ITokenizer tokenizer = new ClassicTokenizer(true);
SentenceSegmenter segmenter = new TokenBasedSentenceSegmenter(tokenizer);
var sentences = segmenter.GetSentences(paragraph);
foreach (string sentence in sentences)
{
Console.WriteLine(sentence);
}
}
public TokenStreamComponentsAnonymousInnerClassHelper(ClassicAnalyzer outerInstance, ClassicTokenizer src, TokenStream tok, Reader reader)
: base(src, tok)
{
this.outerInstance = outerInstance;
this.reader = reader;
this.src = src;
}
public override TokenStreamComponents CreateComponents(string fieldName, Reader reader)
{
var src = new ClassicTokenizer(matchVersion, reader);
src.MaxTokenLength = maxTokenLength;
TokenStream tok = new ClassicFilter(src);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponentsAnonymousInnerClassHelper(this, src, tok, reader);
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
//ORIGINAL LINE: @Override protected TokenStreamComponents createComponents(final String fieldName, final java.io.Reader reader)
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final ClassicTokenizer src = new ClassicTokenizer(matchVersion, reader);
ClassicTokenizer src = new ClassicTokenizer(matchVersion, reader);
src.MaxTokenLength = maxTokenLength;
TokenStream tok = new ClassicFilter(src);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponentsAnonymousInnerClassHelper(this, src, tok, reader);
}
public override ClassicTokenizer create(AttributeFactory factory, Reader input)
{
ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory, input);
tokenizer.MaxTokenLength = maxTokenLength;
return tokenizer;
}
