private string Convert(string phrase, TokenizerMapConverter tokenizerMapConverter) { var tokenizer = TokenizerFactory.Create(tokenizerMapConverter.Tokenizer); var words = tokenizer.Segment(phrase); var wordsConvert = WordsConverterFactory.Create(tokenizerMapConverter.Converter); return(wordsConvert.Convert(words)); }
public void TestList(string ch, AsmToken[] tk) { var sut = TokenizerFactory.Create(); var actual = sut.Tokenize(ch) .Select(t => t.Kind) .ToList(); Assert.Equal(tk, actual); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tf = tokenizer.Create(reader); if (tokenfilter != null) { return(new TokenStreamComponents(tf, tokenfilter.Create(tf))); } else { return(new TokenStreamComponents(tf)); } }
private string Convert(string phrase, TokenizerMapConverter tokenizerMapConverter) { //当Tokenizer没有配置时默认返回原始字符串 if (tokenizerMapConverter != null) { var tokenizer = TokenizerFactory.Create(tokenizerMapConverter.Tokenizer); var words = tokenizer.Segment(phrase); var wordsConvert = WordsConverterFactory.Create(tokenizerMapConverter.Converter); return(wordsConvert.Convert(words)); } else { return(phrase); } }
private static TokenStream LoadTokenizer(TokenizerFactory tokFactory, TextReader reader) { return(tokFactory.Create(reader)); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { #pragma warning disable 612, 618 Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(LuceneVersion.LUCENE_CURRENT, reader) : factory.Create(reader); TokenStream stream = outerInstance.ignoreCase ? (TokenStream) new LowerCaseFilter(LuceneVersion.LUCENE_CURRENT, tokenizer) : tokenizer; #pragma warning restore 612, 618 return(new TokenStreamComponents(tokenizer, stream)); }
protected T Parse(string source) { return(Parser.Parse(TokenizerFactory.Create().Tokenize(source))); }
public virtual void Test() { IList <Type> analysisClasses = typeof(StandardAnalyzer).Assembly.GetTypes() .Where(c => { var typeInfo = c; return(!typeInfo.IsAbstract && typeInfo.IsPublic && !typeInfo.IsInterface && typeInfo.IsClass && (typeInfo.GetCustomAttribute <ObsoleteAttribute>() == null) && !testComponents.Contains(c) && !crazyComponents.Contains(c) && !oddlyNamedComponents.Contains(c) && !deprecatedDuplicatedComponents.Contains(c) && (typeInfo.IsSubclassOf(typeof(Tokenizer)) || typeInfo.IsSubclassOf(typeof(TokenFilter)) || typeInfo.IsSubclassOf(typeof(CharFilter)))); }) .ToList(); foreach (Type c in analysisClasses) { IDictionary <string, string> args = new Dictionary <string, string>(); args["luceneMatchVersion"] = TEST_VERSION_CURRENT.ToString(); if (c.IsSubclassOf(typeof(Tokenizer))) { string clazzName = c.Name; assertTrue(clazzName.EndsWith("Tokenizer", StringComparison.Ordinal)); string simpleName = clazzName.Substring(0, clazzName.Length - 9); assertNotNull(TokenizerFactory.LookupClass(simpleName)); TokenizerFactory instance = null; try { instance = TokenizerFactory.ForName(simpleName, args); assertNotNull(instance); if (instance is IResourceLoaderAware resourceLoaderAware) { resourceLoaderAware.Inform(loader); } assertSame(c, instance.Create(new StringReader("")).GetType()); } catch (Exception e) when(e.IsIllegalArgumentException()) { if (e.InnerException.IsNoSuchMethodException()) { // there is no corresponding ctor available throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } // TODO: For now pass because some factories have not yet a default config that always works } } else if (c.IsSubclassOf(typeof(TokenFilter))) { string clazzName = c.Name; assertTrue(clazzName.EndsWith("Filter", StringComparison.Ordinal)); string simpleName = clazzName.Substring(0, clazzName.Length - (clazzName.EndsWith("TokenFilter", StringComparison.Ordinal) ? 11 : 6)); assertNotNull(TokenFilterFactory.LookupClass(simpleName)); TokenFilterFactory instance = null; try { instance = TokenFilterFactory.ForName(simpleName, args); assertNotNull(instance); if (instance is IResourceLoaderAware resourceLoaderAware) { resourceLoaderAware.Inform(loader); } Type createdClazz = instance.Create(new KeywordTokenizer(new StringReader(""))).GetType(); // only check instance if factory have wrapped at all! if (typeof(KeywordTokenizer) != createdClazz) { assertSame(c, createdClazz); } } catch (Exception e) when(e.IsIllegalArgumentException()) { if (e.InnerException.IsNoSuchMethodException()) { // there is no corresponding ctor available throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } // TODO: For now pass because some factories have not yet a default config that always works } } else if (c.IsSubclassOf(typeof(CharFilter))) { string clazzName = c.Name; assertTrue(clazzName.EndsWith("CharFilter", StringComparison.Ordinal)); string simpleName = clazzName.Substring(0, clazzName.Length - 10); assertNotNull(CharFilterFactory.LookupClass(simpleName)); CharFilterFactory instance = null; try { instance = CharFilterFactory.ForName(simpleName, args); assertNotNull(instance); if (instance is IResourceLoaderAware resourceLoaderAware) { resourceLoaderAware.Inform(loader); } Type createdClazz = instance.Create(new StringReader("")).GetType(); // only check instance if factory have wrapped at all! if (typeof(StringReader) != createdClazz) { assertSame(c, createdClazz); } } catch (Exception e) when(e.IsIllegalArgumentException()) { if (e.InnerException.IsNoSuchMethodException()) { // there is no corresponding ctor available throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details) } // TODO: For now pass because some factories have not yet a default config that always works } } } }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(LuceneVersion.LUCENE_CURRENT, reader) : factory.Create(reader); TokenStream stream = outerInstance.ignoreCase ? new LowerCaseFilter(LuceneVersion.LUCENE_CURRENT, tokenizer) : tokenizer; return(new Analyzer.TokenStreamComponents(tokenizer, stream)); }
public void Inform(IResourceLoader loader) { TokenizerFactory factory = tokenizerFactory is null ? null : LoadTokenizerFactory(loader, tokenizerFactory); Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { #pragma warning disable 612, 618 Tokenizer tokenizer = factory is null ? new WhitespaceTokenizer(LuceneVersion.LUCENE_CURRENT, reader) : factory.Create(reader); TokenStream stream = ignoreCase ? (TokenStream) new LowerCaseFilter(LuceneVersion.LUCENE_CURRENT, tokenizer) : tokenizer; #pragma warning restore 612, 618 return(new TokenStreamComponents(tokenizer, stream)); }); try { string formatClass = format; if (format is null || format.Equals("solr", StringComparison.Ordinal)) { formatClass = typeof(SolrSynonymParser).AssemblyQualifiedName; } else if (format.Equals("wordnet", StringComparison.Ordinal)) { formatClass = typeof(WordnetSynonymParser).AssemblyQualifiedName; } // TODO: expose dedup as a parameter? map = LoadSynonyms(loader, formatClass, true, analyzer); }
public virtual void Test() { IList <Type> analysisClasses = new List <Type>( typeof(StandardAnalyzer).Assembly.GetTypes() .Where(c => !c.IsAbstract && c.IsPublic && !c.IsInterface && c.IsClass && (c.GetCustomAttribute <ObsoleteAttribute>() == null) && !testComponents.Contains(c) && !crazyComponents.Contains(c) && !oddlyNamedComponents.Contains(c) && !deprecatedDuplicatedComponents.Contains(c) && (c.IsSubclassOf(typeof(Tokenizer)) || c.IsSubclassOf(typeof(TokenFilter)) || c.IsSubclassOf(typeof(CharFilter))) )); foreach (Type c in analysisClasses) { IDictionary <string, string> args = new Dictionary <string, string>(); args["luceneMatchVersion"] = TEST_VERSION_CURRENT.ToString(); if (c.IsSubclassOf(typeof(Tokenizer))) { string clazzName = c.Name; assertTrue(clazzName.EndsWith("Tokenizer", StringComparison.Ordinal)); string simpleName = clazzName.Substring(0, clazzName.Length - 9); assertNotNull(TokenizerFactory.LookupClass(simpleName)); TokenizerFactory instance = null; try { instance = TokenizerFactory.ForName(simpleName, args); assertNotNull(instance); if (instance is IResourceLoaderAware) { ((IResourceLoaderAware)instance).Inform(loader); } assertSame(c, instance.Create(new StringReader("")).GetType()); } catch (System.ArgumentException e) { if (e.InnerException is MissingMethodException) { // there is no corresponding ctor available throw e; } // TODO: For now pass because some factories have not yet a default config that always works } } else if (c.IsSubclassOf(typeof(TokenFilter))) { string clazzName = c.Name; assertTrue(clazzName.EndsWith("Filter", StringComparison.Ordinal)); string simpleName = clazzName.Substring(0, clazzName.Length - (clazzName.EndsWith("TokenFilter", StringComparison.Ordinal) ? 11 : 6)); assertNotNull(TokenFilterFactory.LookupClass(simpleName)); TokenFilterFactory instance = null; try { instance = TokenFilterFactory.ForName(simpleName, args); assertNotNull(instance); if (instance is IResourceLoaderAware) { ((IResourceLoaderAware)instance).Inform(loader); } Type createdClazz = instance.Create(new KeywordTokenizer(new StringReader(""))).GetType(); // only check instance if factory have wrapped at all! if (typeof(KeywordTokenizer) != createdClazz) { assertSame(c, createdClazz); } } catch (System.ArgumentException e) { if (e.InnerException is MissingMethodException) { // there is no corresponding ctor available throw e; } // TODO: For now pass because some factories have not yet a default config that always works } } else if (c.IsSubclassOf(typeof(CharFilter))) { string clazzName = c.Name; assertTrue(clazzName.EndsWith("CharFilter", StringComparison.Ordinal)); string simpleName = clazzName.Substring(0, clazzName.Length - 10); assertNotNull(CharFilterFactory.LookupClass(simpleName)); CharFilterFactory instance = null; try { instance = CharFilterFactory.ForName(simpleName, args); assertNotNull(instance); if (instance is IResourceLoaderAware) { ((IResourceLoaderAware)instance).Inform(loader); } Type createdClazz = instance.Create(new StringReader("")).GetType(); // only check instance if factory have wrapped at all! if (typeof(StringReader) != createdClazz) { assertSame(c, createdClazz); } } catch (System.ArgumentException e) { if (e.InnerException is MissingMethodException) { // there is no corresponding ctor available throw e; } // TODO: For now pass because some factories have not yet a default config that always works } } } }
private static Tokenizer <LangToken> CreateSut() { return(TokenizerFactory.Create()); }
//[InlineData("ADD R1,R2")] //[InlineData("ADD R1,R2,R3")] public void Add(string source) { var tokenList = TokenizerFactory.Create().Tokenize(source); var commands = Parser.Parser.AsmParser.Parse(tokenList); }
public void Add(string source) { var tokenList = TokenizerFactory.Create().Tokenize(source); var address = Parser.Parser.MemoryAddress.Parse(tokenList); }