C# (CSharp) WordTokenizerOptions Examples

Example #1

        private static bool IsFlagOn(WordTokenizerOptions allFlags, WordTokenizerOptions singleFlag)
        {
            var f = (int)singleFlag;
            var r = (int)allFlags & f;

            return(r == f);
        }

Example #2

File: Program.cs Project: yassersouri/Corpus-Builder

        static WordPatternInfo[] SeparateWords(string sentence)
        {
            WordTokenizerOptions wto = WordTokenizerOptions.ReturnPunctuations;
            WordTokenizer        st  = new WordTokenizer(wto);

            WordPatternInfo[] wpi = st.ExtractWords(sentence).ToArray();
            return(wpi);
        }

Example #3

File: WordTokenizer.cs Project: sinairv/PdfRenamer

 public WordTokenizer(WordTokenizerOptions options)
 {
     // read options
     m_retPuncs = IsFlagOn(options, WordTokenizerOptions.ReturnPunctuations);
     m_retWs = IsFlagOn(options, WordTokenizerOptions.ReturnWhitespaces);
     m_retWsCharByChar = IsFlagOn(options, WordTokenizerOptions.ReturnWhitespacesCharacterByCharacter);
     m_isNumAr1Word = IsFlagOn(options, WordTokenizerOptions.TreatNumberArabicCharCombinationAsOneWords);
     m_isNumNonAr1Word = IsFlagOn(options, WordTokenizerOptions.TreatNumberNonArabicCharCombinationAsOneWords);
     m_isArNonAr1Word = IsFlagOn(options, WordTokenizerOptions.TreatArabicNonArabicCharCombinationAsOneWords);
 }

Example #4

 public WordTokenizer(WordTokenizerOptions options)
 {
     // read options
     m_retPuncs        = IsFlagOn(options, WordTokenizerOptions.ReturnPunctuations);
     m_retWs           = IsFlagOn(options, WordTokenizerOptions.ReturnWhitespaces);
     m_retWsCharByChar = IsFlagOn(options, WordTokenizerOptions.ReturnWhitespacesCharacterByCharacter);
     m_isNumAr1Word    = IsFlagOn(options, WordTokenizerOptions.TreatNumberArabicCharCombinationAsOneWords);
     m_isNumNonAr1Word = IsFlagOn(options, WordTokenizerOptions.TreatNumberNonArabicCharCombinationAsOneWords);
     m_isArNonAr1Word  = IsFlagOn(options, WordTokenizerOptions.TreatArabicNonArabicCharCombinationAsOneWords);
 }

Example #5

File: WordTokenizer.cs Project: sinairv/PdfRenamer

 private static bool IsFlagOn(WordTokenizerOptions allFlags, WordTokenizerOptions singleFlag)
 {
     var f = (int) singleFlag;
     var r = (int) allFlags & f;
     return (r == f);
 }