public virtual void TestLowerCaseTokenizer() { StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest"); LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, reader); AssertTokenStreamContents(tokenizer, new string[] { "tokenizer", "\ud801\udc44test" }); }
public virtual void TestLowerCaseTokenizerBWCompat() { StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest"); LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(LuceneVersion.LUCENE_30, reader); AssertTokenStreamContents(tokenizer, new string[] { "tokenizer", "test" }); }
/// <summary> /// Creates /// <see cref="TokenStreamComponents"/> /// used to tokenize all the text in the provided <see cref="TextReader"/>. /// </summary> /// <returns> <see cref="TokenStreamComponents"/> /// built from a <see cref="LowerCaseTokenizer"/> filtered with /// <see cref="StopFilter"/> </returns> protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer source = new LowerCaseTokenizer(m_matchVersion, reader); return(new TokenStreamComponents(source, new StopFilter(m_matchVersion, source, m_stopwords))); }
/// <summary> /// Creates /// <seealso cref="Analyzer.TokenStreamComponents"/> /// used to tokenize all the text in the provided <seealso cref="TextReader"/>. /// </summary> /// <returns> <seealso cref="Analyzer.TokenStreamComponents"/> /// built from a <seealso cref="LowerCaseTokenizer"/> filtered with /// <seealso cref="StopFilter"/> </returns> public override Analyzer.TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer source = new LowerCaseTokenizer(matchVersion, reader); return(new Analyzer.TokenStreamComponents(source, new StopFilter(matchVersion, source, stopwords))); }
/// <summary> /// Creates /// <seealso cref="Analyzer.TokenStreamComponents"/> /// used to tokenize all the text in the provided <seealso cref="TextReader"/>. /// </summary> /// <returns> <seealso cref="Analyzer.TokenStreamComponents"/> /// built from a <seealso cref="LowerCaseTokenizer"/> filtered with /// <seealso cref="StopFilter"/> </returns> public override Analyzer.TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer source = new LowerCaseTokenizer(matchVersion, reader); return new Analyzer.TokenStreamComponents(source, new StopFilter(matchVersion, source, stopwords)); }
public virtual void TestLowerCaseTokenizerBWCompat() { StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest"); LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(LuceneVersion.LUCENE_30, reader); AssertTokenStreamContents(tokenizer, new string[] { "tokenizer", "test" }); }
public virtual void TestLowerCaseTokenizer() { StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest"); LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, reader); AssertTokenStreamContents(tokenizer, new string[] { "tokenizer", "\ud801\udc44test" }); }