public void TestIgnoreAccentsButNotCase()
{
String withAccents = "résumé";
String withoutAccents = "resume";
String withAccentsUpperCase = "Résumé";
String withoutAccentsUpperCase = "Resume";
TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
"locale", "en",
"strength", "primary",
"caseLevel", "true");
TokenStream tsWithAccents = factory.Create(
new KeywordTokenizer(new StringReader(withAccents)));
TokenStream tsWithoutAccents = factory.Create(
new KeywordTokenizer(new StringReader(withoutAccents)));
assertCollatesToSame(tsWithAccents, tsWithoutAccents);
TokenStream tsWithAccentsUpperCase = factory.Create(
new KeywordTokenizer(new StringReader(withAccentsUpperCase)));
TokenStream tsWithoutAccentsUpperCase = factory.Create(
new KeywordTokenizer(new StringReader(withoutAccentsUpperCase)));
assertCollatesToSame(tsWithAccentsUpperCase, tsWithoutAccentsUpperCase);
// now assert that case still matters: resume < Resume
TokenStream tsLower = factory.Create(
new KeywordTokenizer(new StringReader(withoutAccents)));
TokenStream tsUpper = factory.Create(
new KeywordTokenizer(new StringReader(withoutAccentsUpperCase)));
assertCollation(tsLower, tsUpper, -1);
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tf = tokenizer.Create(reader);
if (tokenfilter != null)
{
return(new TokenStreamComponents(tf, tokenfilter.Create(tf)));
}
else
{
return(new TokenStreamComponents(tf));
}
}
public void TestIgnoreWhitespace()
{
String withSpace = "foo bar";
String withoutSpace = "foobar";
String withPunctuation = "foo-bar";
TokenFilterFactory factory = tokenFilterFactory("ICUCollationKey",
"locale", "en",
"strength", "primary",
"alternate", "shifted",
"variableTop", " ");
TokenStream tsWithSpace = factory.Create(
new KeywordTokenizer(new StringReader(withSpace)));
TokenStream tsWithoutSpace = factory.Create(
new KeywordTokenizer(new StringReader(withoutSpace)));
assertCollatesToSame(tsWithSpace, tsWithoutSpace);
// now assert that punctuation still matters: foo-bar < foo bar
tsWithSpace = factory.Create(
new KeywordTokenizer(new StringReader(withSpace)));
TokenStream tsWithPunctuation = factory.Create(
new KeywordTokenizer(new StringReader(withPunctuation)));
assertCollation(tsWithPunctuation, tsWithSpace, -1);
}
public override TokenStream Create(TokenStream input)
{
return(delegator.Create(input));
}
public virtual void Test()
{
IList <Type> analysisClasses = typeof(StandardAnalyzer).Assembly.GetTypes()
.Where(c =>
{
var typeInfo = c;
return(!typeInfo.IsAbstract && typeInfo.IsPublic && !typeInfo.IsInterface && typeInfo.IsClass && (typeInfo.GetCustomAttribute <ObsoleteAttribute>() == null) &&
!testComponents.Contains(c) && !crazyComponents.Contains(c) && !oddlyNamedComponents.Contains(c) && !deprecatedDuplicatedComponents.Contains(c) &&
(typeInfo.IsSubclassOf(typeof(Tokenizer)) || typeInfo.IsSubclassOf(typeof(TokenFilter)) || typeInfo.IsSubclassOf(typeof(CharFilter))));
})
.ToList();
foreach (Type c in analysisClasses)
{
IDictionary <string, string> args = new Dictionary <string, string>();
args["luceneMatchVersion"] = TEST_VERSION_CURRENT.ToString();
if (c.IsSubclassOf(typeof(Tokenizer)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Tokenizer", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 9);
assertNotNull(TokenizerFactory.LookupClass(simpleName));
TokenizerFactory instance = null;
try
{
instance = TokenizerFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware resourceLoaderAware)
{
resourceLoaderAware.Inform(loader);
}
assertSame(c, instance.Create(new StringReader("")).GetType());
}
catch (Exception e) when(e.IsIllegalArgumentException())
{
if (e.InnerException.IsNoSuchMethodException())
{
// there is no corresponding ctor available
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(TokenFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Filter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - (clazzName.EndsWith("TokenFilter", StringComparison.Ordinal) ? 11 : 6));
assertNotNull(TokenFilterFactory.LookupClass(simpleName));
TokenFilterFactory instance = null;
try
{
instance = TokenFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware resourceLoaderAware)
{
resourceLoaderAware.Inform(loader);
}
Type createdClazz = instance.Create(new KeywordTokenizer(new StringReader(""))).GetType();
// only check instance if factory have wrapped at all!
if (typeof(KeywordTokenizer) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (Exception e) when(e.IsIllegalArgumentException())
{
if (e.InnerException.IsNoSuchMethodException())
{
// there is no corresponding ctor available
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(CharFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("CharFilter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 10);
assertNotNull(CharFilterFactory.LookupClass(simpleName));
CharFilterFactory instance = null;
try
{
instance = CharFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware resourceLoaderAware)
{
resourceLoaderAware.Inform(loader);
}
Type createdClazz = instance.Create(new StringReader("")).GetType();
// only check instance if factory have wrapped at all!
if (typeof(StringReader) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (Exception e) when(e.IsIllegalArgumentException())
{
if (e.InnerException.IsNoSuchMethodException())
{
// there is no corresponding ctor available
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
}
}
public virtual void Test()
{
IList <Type> analysisClasses = new List <Type>(
typeof(StandardAnalyzer).Assembly.GetTypes()
.Where(c => !c.IsAbstract && c.IsPublic && !c.IsInterface && c.IsClass && (c.GetCustomAttribute <ObsoleteAttribute>() == null) &&
!testComponents.Contains(c) && !crazyComponents.Contains(c) && !oddlyNamedComponents.Contains(c) && !deprecatedDuplicatedComponents.Contains(c) &&
(c.IsSubclassOf(typeof(Tokenizer)) || c.IsSubclassOf(typeof(TokenFilter)) || c.IsSubclassOf(typeof(CharFilter)))
));
foreach (Type c in analysisClasses)
{
IDictionary <string, string> args = new Dictionary <string, string>();
args["luceneMatchVersion"] = TEST_VERSION_CURRENT.ToString();
if (c.IsSubclassOf(typeof(Tokenizer)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Tokenizer", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 9);
assertNotNull(TokenizerFactory.LookupClass(simpleName));
TokenizerFactory instance = null;
try
{
instance = TokenizerFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware)
{
((IResourceLoaderAware)instance).Inform(loader);
}
assertSame(c, instance.Create(new StringReader("")).GetType());
}
catch (System.ArgumentException e)
{
if (e.InnerException is MissingMethodException)
{
// there is no corresponding ctor available
throw e;
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(TokenFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Filter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - (clazzName.EndsWith("TokenFilter", StringComparison.Ordinal) ? 11 : 6));
assertNotNull(TokenFilterFactory.LookupClass(simpleName));
TokenFilterFactory instance = null;
try
{
instance = TokenFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware)
{
((IResourceLoaderAware)instance).Inform(loader);
}
Type createdClazz = instance.Create(new KeywordTokenizer(new StringReader(""))).GetType();
// only check instance if factory have wrapped at all!
if (typeof(KeywordTokenizer) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (System.ArgumentException e)
{
if (e.InnerException is MissingMethodException)
{
// there is no corresponding ctor available
throw e;
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(CharFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("CharFilter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 10);
assertNotNull(CharFilterFactory.LookupClass(simpleName));
CharFilterFactory instance = null;
try
{
instance = CharFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware)
{
((IResourceLoaderAware)instance).Inform(loader);
}
Type createdClazz = instance.Create(new StringReader("")).GetType();
// only check instance if factory have wrapped at all!
if (typeof(StringReader) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (System.ArgumentException e)
{
if (e.InnerException is MissingMethodException)
{
// there is no corresponding ctor available
throw e;
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
}
}