public virtual ITokenizer Create(TokenizationOptions options) { var tokenizer = CreateTokenizer(options.TokenizerKind); tokenizer.Configure(options); return(tokenizer); }
public virtual ITokenizer Create(TokenizationOptions options) { if (options is null) { throw new ArgumentNullException(nameof(options)); } var tokenizer = CreateTokenizer(options.TokenizerKind); tokenizer.Configure(options); return(tokenizer); }
public List <Token> Tokenize(string sentence, TokenizationOptions options) { Init(); var tokens = segmenter.Cut(sentence) .Select(x => new Token { Text = x }).ToList(); CorrectTokenPosition(sentence, tokens); return(tokens); }
/// <summary> /// Tokenizes a string, returning its list of words. /// </summary> /// <param name="text">The document.</param> /// <param name="options">The tokenization options.</param> /// <param name="punctuationCharacters">The characters considered as punctuation.</param> /// <returns>The tokens.</returns> public static string[] Tokenize( string text, TokenizationOptions options = TokenizationOptions.All, string punctuationCharacters = PunctuationCharacters) { var left = "⋘"; var right = "⋙"; if ((options & TokenizationOptions.StripHtml) != 0) { text = Regex.Replace(text, "<[^<>]+>", string.Empty); } if ((options & TokenizationOptions.StripNumbers) != 0) { text = Regex.Replace(text, "[0-9]+", left + "number" + right); } if ((options & TokenizationOptions.StripUrls) != 0) { text = Regex.Replace(text, @"(http|https)://[^\s]*", left + "httpaddr" + right); } if ((options & TokenizationOptions.StripEmailAddresses) != 0) { text = Regex.Replace(text, @"[^\s]+@[^\s]+", left + "emailaddr" + right); } if ((options & TokenizationOptions.StripMonetary) != 0) { text = Regex.Replace(text, "[$]+", left + "dollar" + right); } if ((options & TokenizationOptions.StripUsernames) != 0) { text = Regex.Replace(text, @"@[^\s]+", left + "username" + right); } // Tokenize and also get rid of any punctuation var tokens = text.Split(punctuationCharacters.ToCharArray()).Select( token => { var result = token.Replace(left, "{"); result = result.Replace(right, "}"); return(result); }).Where(str => !string.IsNullOrWhiteSpace(str)).ToArray(); return(tokens); }
/// <summary> /// Initializes a new instance of the <see cref="Tokenizer"/> class. /// </summary> /// <param name="tokenizationOptions">The tokenization options for this instance.</param> public Tokenizer(TokenizationOptions tokenizationOptions) { this.Options = tokenizationOptions ?? throw new ArgumentNullException(nameof(tokenizationOptions)); if (tokenizationOptions.Stemming) { this.stemmer = new PorterStemmer(); } this.additionalSplitChars = tokenizationOptions.AdditionalSplitCharacters.Count > 0 ? new HashSet <char>(tokenizationOptions.AdditionalSplitCharacters) : null; this.inputPreprocessorPipeline = new InputPreprocessorPipeline(tokenizationOptions); }
/// <summary> /// Initializes a new instance of the <see cref="InputPreprocessorPipeline"/> class. /// </summary> public InputPreprocessorPipeline(TokenizationOptions options) { if (options is null) { throw new ArgumentNullException(nameof(options)); } if (options.AccentInsensitive) { this.inputPreprocessors.Add(new LatinCharacterNormalizer()); } if (options.CaseInsensitive) { this.inputPreprocessors.Add(new CaseInsensitiveNormalizer()); } }
public FakeTokenizer(TokenizationOptions options) { this.Options = options; }
public void Configure(TokenizationOptions options) { }
public TokenExApiController( TokenizationOptions tokenizationOptions) { _tokenizationOptions = tokenizationOptions; }
protected TokenExGateway(TokenizationOptions tokenizationOptions) => _tokenizationOptions = tokenizationOptions;