protected internal override Analyzer.TokenStreamComponents createComponents(string fieldName, Reader reader)
{
Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return(new Analyzer.TokenStreamComponents(source, new PortugueseLightStemFilter(result)));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testElision() throws Exception
public virtual void testElision()
{
string test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(test));
CharArraySet articles = new CharArraySet(TEST_VERSION_CURRENT, asSet("l", "M"), false);
TokenFilter filter = new ElisionFilter(tokenizer, articles);
IList<string> tas = filter(filter);
assertEquals("embrouille", tas[4]);
assertEquals("O'brian", tas[6]);
assertEquals("enfin", tas[7]);
}
protected internal override TokenStreamComponents CreateComponents(string fieldName)
{
StandardTokenizer src = new StandardTokenizer();
src.MaxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
TokenStream tok = new StandardFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
tok = new ASCIIFoldingFilter(tok);
return(new TokenStreamComponents(src, tok));
}
/// <summary>
/// Creates a
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// which tokenizes all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> A
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from an <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="EnglishPossessiveFilter"/>,
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>
/// , <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided and <seealso cref="PorterStemFilter"/>. </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
// prior to this we get the classic behavior, standardfilter does it for us.
if (matchVersion.onOrAfter(Version.LUCENE_31))
{
result = new EnglishPossessiveFilter(matchVersion, result);
}
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (!stemExclusionSet.Empty)
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new PorterStemFilter(result);
return(new TokenStreamComponents(source, result));
}
/// <summary>
/// Creates
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// used to tokenize all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from a <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="ArabicNormalizationFilter"/>,
/// <seealso cref="PersianNormalizationFilter"/> and Persian Stop words </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source;
Tokenizer source;
if (matchVersion.onOrAfter(Version.LUCENE_31))
{
source = new StandardTokenizer(matchVersion, reader);
}
else
{
source = new ArabicLetterTokenizer(matchVersion, reader);
}
TokenStream result = new LowerCaseFilter(matchVersion, source);
result = new ArabicNormalizationFilter(result);
/* additional persian-specific normalization */
result = new PersianNormalizationFilter(result);
/*
* the order here is important: the stopword list is normalized with the
* above!
*/
return new TokenStreamComponents(source, new StopFilter(matchVersion, result, stopwords));
}
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseStemFilter(result));
}
/// <summary>
/// Creates a
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// which tokenizes all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> A
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from an <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>
/// , <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided and <seealso cref="SnowballFilter"/>. </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (!stemExclusionSet.Empty)
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new SnowballFilter(result, new DanishStemmer());
return new TokenStreamComponents(source, result);
}
/// <summary>
/// Creates a
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// which tokenizes all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> A
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from an <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="EnglishPossessiveFilter"/>,
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>
/// , <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided and <seealso cref="PorterStemFilter"/>. </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
// prior to this we get the classic behavior, standardfilter does it for us.
if (matchVersion.onOrAfter(Version.LUCENE_31))
{
result = new EnglishPossessiveFilter(matchVersion, result);
}
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (!stemExclusionSet.Empty)
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new PorterStemFilter(result);
return new TokenStreamComponents(source, result);
}
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
return new TokenStreamComponents(t, new CJKBigramFilter(t, CJKBigramFilter.HAN, true));
}
public TokenStreamComponentsAnonymousInnerClassHelper(StandardAnalyzer outerInstance, org.apache.lucene.analysis.standard.StandardTokenizer src, TokenStream tok, Reader reader) : base(src, tok)
{
this.outerInstance = outerInstance;
this.reader = reader;
this.src = src;
}
/// <summary>
/// Creates
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// used to tokenize all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from a <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>
/// , <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided, <seealso cref="GermanNormalizationFilter"/> and <seealso cref="GermanLightStemFilter"/> </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
result = new SetKeywordMarkerFilter(result, exclusionSet);
if (matchVersion.onOrAfter(Version.LUCENE_36))
{
result = new GermanNormalizationFilter(result);
result = new GermanLightStemFilter(result);
}
else if (matchVersion.onOrAfter(Version.LUCENE_31))
{
result = new SnowballFilter(result, new German2Stemmer());
}
else
{
result = new GermanStemFilter(result);
}
return new TokenStreamComponents(source, result);
}
/// <summary>
/// Creates a
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// which tokenizes all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> A
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from an <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="IrishLowerCaseFilter"/>, <seealso cref="StopFilter"/>
/// , <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided and <seealso cref="SnowballFilter"/>. </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
StopFilter s = new StopFilter(matchVersion, result, HYPHENATIONS);
if (!matchVersion.onOrAfter(Version.LUCENE_44))
{
s.EnablePositionIncrements = false;
}
result = s;
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new IrishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
if (!stemExclusionSet.Empty)
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new SnowballFilter(result, new IrishStemmer());
return new TokenStreamComponents(source, result);
}
/// <summary>
/// Returns a (possibly reused) <seealso cref="TokenStream"/> which tokenizes all the
/// text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> A <seealso cref="TokenStream"/> built from a <seealso cref="StandardTokenizer"/>
/// filtered with <seealso cref="StandardFilter"/>, <seealso cref="LowerCaseFilter"/>,
/// <seealso cref="StopFilter"/>, <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is provided,
/// <seealso cref="StemmerOverrideFilter"/>, and <seealso cref="SnowballFilter"/> </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader aReader)
{
if (matchVersion.onOrAfter(Version.LUCENE_31))
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, aReader);
Tokenizer source = new StandardTokenizer(matchVersion, aReader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stoptable);
if (!excltable.Empty)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
if (stemdict != null)
{
result = new StemmerOverrideFilter(result, stemdict);
}
result = new SnowballFilter(result, new org.tartarus.snowball.ext.DutchStemmer());
return new TokenStreamComponents(source, result);
}
else
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, aReader);
Tokenizer source = new StandardTokenizer(matchVersion, aReader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, stoptable);
if (!excltable.Empty)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
result = new DutchStemFilter(result, origStemdict);
return new TokenStreamComponents(source, result);
}
}
/// <summary>
/// Creates
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// used to tokenize all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from a <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="IndicNormalizationFilter"/>,
/// <seealso cref="HindiNormalizationFilter"/>, <seealso cref="SetKeywordMarkerFilter"/>
/// if a stem exclusion set is provided, <seealso cref="HindiStemFilter"/>, and
/// Hindi Stop words </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source;
Tokenizer source;
if (matchVersion.onOrAfter(Version.LUCENE_36))
{
source = new StandardTokenizer(matchVersion, reader);
}
else
{
source = new IndicTokenizer(matchVersion, reader);
}
TokenStream result = new LowerCaseFilter(matchVersion, source);
if (!stemExclusionSet.Empty)
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new IndicNormalizationFilter(result);
result = new HindiNormalizationFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
result = new HindiStemFilter(result);
return new TokenStreamComponents(source, result);
}
