C# (CSharp) advCalcCore.Tokenizing.Tokenizers CharRange.Contains Examples

Example #1

File: WhitespaceTokenizer.cs Project: fritz3n/advCalcCoreOpen

        public TokenizerResult Tokenize(ITracker tracker)
        {
            int count = 0;

            while (tracker.ReadWithOffset(count, out c) && range.Contains(c))
            {
                count++;
            }
            tracker.AddToken(null, count);
            return(TokenizerResult.Success);
        }

Example #2

        public TokenizerResult Tokenize(ITracker tracker)
        {
            int count = 0;

            while (tracker.ReadWithOffset(count, out c) && range.Contains(c))
            {
                if (c == '\n')                  // Newline is used as Seperator
                {
                    break;
                }

                count++;
            }

            tracker.AddToken(null, count);

            return(TokenizerResult.Success);
        }