private string Convert(string phrase, TokenizerMapConverter tokenizerMapConverter)
{
var tokenizer = TokenizerFactory.Create(tokenizerMapConverter.Tokenizer);
var words = tokenizer.Segment(phrase);
var wordsConvert = WordsConverterFactory.Create(tokenizerMapConverter.Converter);
return(wordsConvert.Convert(words));
}
public void TestList(string ch, AsmToken[] tk)
{
var sut = TokenizerFactory.Create();
var actual = sut.Tokenize(ch)
.Select(t => t.Kind)
.ToList();
Assert.Equal(tk, actual);
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tf = tokenizer.Create(reader);
if (tokenfilter != null)
{
return(new TokenStreamComponents(tf, tokenfilter.Create(tf)));
}
else
{
return(new TokenStreamComponents(tf));
}
}
private string Convert(string phrase, TokenizerMapConverter tokenizerMapConverter)
{
//当Tokenizer没有配置时默认返回原始字符串
if (tokenizerMapConverter != null)
{
var tokenizer = TokenizerFactory.Create(tokenizerMapConverter.Tokenizer);
var words = tokenizer.Segment(phrase);
var wordsConvert = WordsConverterFactory.Create(tokenizerMapConverter.Converter);
return(wordsConvert.Convert(words));
}
else
{
return(phrase);
}
}
private static TokenStream LoadTokenizer(TokenizerFactory tokFactory, TextReader reader)
{
return(tokFactory.Create(reader));
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
#pragma warning disable 612, 618
Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(LuceneVersion.LUCENE_CURRENT, reader) : factory.Create(reader);
TokenStream stream = outerInstance.ignoreCase ? (TokenStream) new LowerCaseFilter(LuceneVersion.LUCENE_CURRENT, tokenizer) : tokenizer;
#pragma warning restore 612, 618
return(new TokenStreamComponents(tokenizer, stream));
}
protected T Parse(string source)
{
return(Parser.Parse(TokenizerFactory.Create().Tokenize(source)));
}
public virtual void Test()
{
IList <Type> analysisClasses = typeof(StandardAnalyzer).Assembly.GetTypes()
.Where(c =>
{
var typeInfo = c;
return(!typeInfo.IsAbstract && typeInfo.IsPublic && !typeInfo.IsInterface && typeInfo.IsClass && (typeInfo.GetCustomAttribute <ObsoleteAttribute>() == null) &&
!testComponents.Contains(c) && !crazyComponents.Contains(c) && !oddlyNamedComponents.Contains(c) && !deprecatedDuplicatedComponents.Contains(c) &&
(typeInfo.IsSubclassOf(typeof(Tokenizer)) || typeInfo.IsSubclassOf(typeof(TokenFilter)) || typeInfo.IsSubclassOf(typeof(CharFilter))));
})
.ToList();
foreach (Type c in analysisClasses)
{
IDictionary <string, string> args = new Dictionary <string, string>();
args["luceneMatchVersion"] = TEST_VERSION_CURRENT.ToString();
if (c.IsSubclassOf(typeof(Tokenizer)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Tokenizer", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 9);
assertNotNull(TokenizerFactory.LookupClass(simpleName));
TokenizerFactory instance = null;
try
{
instance = TokenizerFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware resourceLoaderAware)
{
resourceLoaderAware.Inform(loader);
}
assertSame(c, instance.Create(new StringReader("")).GetType());
}
catch (Exception e) when(e.IsIllegalArgumentException())
{
if (e.InnerException.IsNoSuchMethodException())
{
// there is no corresponding ctor available
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(TokenFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Filter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - (clazzName.EndsWith("TokenFilter", StringComparison.Ordinal) ? 11 : 6));
assertNotNull(TokenFilterFactory.LookupClass(simpleName));
TokenFilterFactory instance = null;
try
{
instance = TokenFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware resourceLoaderAware)
{
resourceLoaderAware.Inform(loader);
}
Type createdClazz = instance.Create(new KeywordTokenizer(new StringReader(""))).GetType();
// only check instance if factory have wrapped at all!
if (typeof(KeywordTokenizer) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (Exception e) when(e.IsIllegalArgumentException())
{
if (e.InnerException.IsNoSuchMethodException())
{
// there is no corresponding ctor available
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(CharFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("CharFilter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 10);
assertNotNull(CharFilterFactory.LookupClass(simpleName));
CharFilterFactory instance = null;
try
{
instance = CharFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware resourceLoaderAware)
{
resourceLoaderAware.Inform(loader);
}
Type createdClazz = instance.Create(new StringReader("")).GetType();
// only check instance if factory have wrapped at all!
if (typeof(StringReader) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (Exception e) when(e.IsIllegalArgumentException())
{
if (e.InnerException.IsNoSuchMethodException())
{
// there is no corresponding ctor available
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
}
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(LuceneVersion.LUCENE_CURRENT, reader) : factory.Create(reader);
TokenStream stream = outerInstance.ignoreCase ? new LowerCaseFilter(LuceneVersion.LUCENE_CURRENT, tokenizer) : tokenizer;
return(new Analyzer.TokenStreamComponents(tokenizer, stream));
}
public void Inform(IResourceLoader loader)
{
TokenizerFactory factory = tokenizerFactory is null ? null : LoadTokenizerFactory(loader, tokenizerFactory);
Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
#pragma warning disable 612, 618
Tokenizer tokenizer = factory is null ? new WhitespaceTokenizer(LuceneVersion.LUCENE_CURRENT, reader) : factory.Create(reader);
TokenStream stream = ignoreCase ? (TokenStream) new LowerCaseFilter(LuceneVersion.LUCENE_CURRENT, tokenizer) : tokenizer;
#pragma warning restore 612, 618
return(new TokenStreamComponents(tokenizer, stream));
});
try
{
string formatClass = format;
if (format is null || format.Equals("solr", StringComparison.Ordinal))
{
formatClass = typeof(SolrSynonymParser).AssemblyQualifiedName;
}
else if (format.Equals("wordnet", StringComparison.Ordinal))
{
formatClass = typeof(WordnetSynonymParser).AssemblyQualifiedName;
}
// TODO: expose dedup as a parameter?
map = LoadSynonyms(loader, formatClass, true, analyzer);
}
public virtual void Test()
{
IList <Type> analysisClasses = new List <Type>(
typeof(StandardAnalyzer).Assembly.GetTypes()
.Where(c => !c.IsAbstract && c.IsPublic && !c.IsInterface && c.IsClass && (c.GetCustomAttribute <ObsoleteAttribute>() == null) &&
!testComponents.Contains(c) && !crazyComponents.Contains(c) && !oddlyNamedComponents.Contains(c) && !deprecatedDuplicatedComponents.Contains(c) &&
(c.IsSubclassOf(typeof(Tokenizer)) || c.IsSubclassOf(typeof(TokenFilter)) || c.IsSubclassOf(typeof(CharFilter)))
));
foreach (Type c in analysisClasses)
{
IDictionary <string, string> args = new Dictionary <string, string>();
args["luceneMatchVersion"] = TEST_VERSION_CURRENT.ToString();
if (c.IsSubclassOf(typeof(Tokenizer)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Tokenizer", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 9);
assertNotNull(TokenizerFactory.LookupClass(simpleName));
TokenizerFactory instance = null;
try
{
instance = TokenizerFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware)
{
((IResourceLoaderAware)instance).Inform(loader);
}
assertSame(c, instance.Create(new StringReader("")).GetType());
}
catch (System.ArgumentException e)
{
if (e.InnerException is MissingMethodException)
{
// there is no corresponding ctor available
throw e;
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(TokenFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("Filter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - (clazzName.EndsWith("TokenFilter", StringComparison.Ordinal) ? 11 : 6));
assertNotNull(TokenFilterFactory.LookupClass(simpleName));
TokenFilterFactory instance = null;
try
{
instance = TokenFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware)
{
((IResourceLoaderAware)instance).Inform(loader);
}
Type createdClazz = instance.Create(new KeywordTokenizer(new StringReader(""))).GetType();
// only check instance if factory have wrapped at all!
if (typeof(KeywordTokenizer) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (System.ArgumentException e)
{
if (e.InnerException is MissingMethodException)
{
// there is no corresponding ctor available
throw e;
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
else if (c.IsSubclassOf(typeof(CharFilter)))
{
string clazzName = c.Name;
assertTrue(clazzName.EndsWith("CharFilter", StringComparison.Ordinal));
string simpleName = clazzName.Substring(0, clazzName.Length - 10);
assertNotNull(CharFilterFactory.LookupClass(simpleName));
CharFilterFactory instance = null;
try
{
instance = CharFilterFactory.ForName(simpleName, args);
assertNotNull(instance);
if (instance is IResourceLoaderAware)
{
((IResourceLoaderAware)instance).Inform(loader);
}
Type createdClazz = instance.Create(new StringReader("")).GetType();
// only check instance if factory have wrapped at all!
if (typeof(StringReader) != createdClazz)
{
assertSame(c, createdClazz);
}
}
catch (System.ArgumentException e)
{
if (e.InnerException is MissingMethodException)
{
// there is no corresponding ctor available
throw e;
}
// TODO: For now pass because some factories have not yet a default config that always works
}
}
}
}
private static Tokenizer <LangToken> CreateSut()
{
return(TokenizerFactory.Create());
}
//[InlineData("ADD R1,R2")]
//[InlineData("ADD R1,R2,R3")]
public void Add(string source)
{
var tokenList = TokenizerFactory.Create().Tokenize(source);
var commands = Parser.Parser.AsmParser.Parse(tokenList);
}
public void Add(string source)
{
var tokenList = TokenizerFactory.Create().Tokenize(source);
var address = Parser.Parser.MemoryAddress.Parse(tokenList);
}
