コード例 #1
0
    private Token CompleteMatch(ISourceStream source) {
      if (source.EOF())
        return null;

      do {
        // Match NewLine
        var lookAhead = source.PreviewChar;
        if (LineTerminators.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          // Treat \r\n as single NewLine
          if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
            source.PreviewPosition++;
          break;
        }

        // Eat up whitespace
        if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          continue;
        }

        // Fail on anything else
        return null;
      }
      while (!source.EOF());

      // Create output token
      return source.CreateToken(this.OutputTerminal);
    }
コード例 #2
0
        private TokenAst ReadToken()
        {
            if (_bufferedTokens.Count > 0)
            {
                TokenAst tkn = _bufferedTokens[0];
                _bufferedTokens.RemoveAt(0);
                return(tkn);
            }

            SkipWhiteSpaces();
            SetTokenStartLocation();

            if (_source.EOF())
            {
                return(TokenAst.Create(LRParser.Eof, _context, _source.TokenStart, string.Empty, LRParser.Eof.Name));
            }

            IEnumerable <ITerminal> terms = SelectTerminals(_source.CurrentChar);
            TokenAst result = MatchTerminals(terms);

            if (result != null && !result.IsError())
            {
                _source.Position = _source.TokenStart.Position + result.Length;
                return(result);
            }

            if (result != null)
            {
                return(result);
            }
            result = LRParser.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar);

            return(result);
        }
コード例 #3
0
        /// <summary>
        /// Tries to match the current position in the <paramref name="source"/> with the rules for a TYPENAME terminal;
        /// returning null if there is no match, returning an  error token if the TYPENAME is malformed or there is no
        /// match and returning a TYPENAME token
        /// otherwise.
        /// </summary>
        /// <param name="context">
        /// The context in which the match is occuring.
        /// </param>
        /// <param name="source">
        /// The source to try to match.
        /// </param>
        /// <returns>
        /// An error token if the TYPENAME is malformed or there is no match and returning a TYPENAME token
        /// otherwise.
        /// </returns>
        public override Token TryMatch([NotNull] ParsingContext context, [NotNull] ISourceStream source)
        {
            if (context == null)
            {
                throw new ArgumentNullException(nameof(context));
            }

            if (source == null)
            {
                throw new ArgumentNullException(nameof(source));
            }

            while (true)
            {
                if (source.EOF())
                {
                    return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
                }

                if (source.PreviewChar == '}' || char.IsWhiteSpace(source.PreviewChar))
                {
                    var token = source.CreateToken(this);
                    if (string.IsNullOrWhiteSpace(token.ValueString))
                    {
                        return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
                    }

                    return(token);
                }

                ++source.PreviewPosition;
            }
        }
コード例 #4
0
        protected override Token QuickParse(ParsingContext context, ISourceStream source)
        {
            if (!_allFirstCharsSet.Contains(source.PreviewChar))
            {
                return(null);
            }
            source.PreviewPosition++;
            while (_allCharsSet.Contains(source.PreviewChar) && !source.EOF())
            {
                source.PreviewPosition++;
            }
            //if it is not a terminator then cancel; we need to go through full algorithm
            if (!this.Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
            {
                return(null);
            }
            var token = source.CreateToken(this.OutputTerminal);

            if (CaseRestriction != CaseRestriction.None && !CheckCaseRestriction(token.ValueString))
            {
                return(null);
            }
            //!!! Do not convert to common case (all-lower) for case-insensitive grammar. Let identifiers remain as is,
            //  it is responsibility of interpreter to provide case-insensitive read/write operations for identifiers
            // if (!this.GrammarData.Grammar.CaseSensitive)
            //    token.Value = token.Text.ToLower(CultureInfo.InvariantCulture);
            CheckReservedWord(token);
            return(token);
        }
コード例 #5
0
        public override void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                case '\r':
                case '\v':
                case ' ':
                case '\t':
                    break;

                case '\n':
                    if (source.NextPreviewChar != ' ' &&
                        source.NextPreviewChar != '\t' &&
                        source.NextPreviewChar != '\n' &&
                        source.NextPreviewChar != '\r' &&
                        source.NextPreviewChar != '\v' &&
                        source.NextPreviewChar != ';')
                    {
                        return;
                    }
                    break;

                default:
                    return;
                }

                source.PreviewPosition++;
            }
        }
コード例 #6
0
ファイル: CommentTerminal.cs プロジェクト: eightrivers/SSharp
    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(_startSymbol, false)) return null;
      source.Position += _startSymbol.Length;

      while (!source.EOF())
      {
        int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

        if (firstCharPos < 0)
        {
          source.Position = source.Text.Length;

          if (_isLineComment)
            return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
          else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
        }
      
        source.Position = firstCharPos;
        if (source.MatchSymbol(_endSymbol, false))
        {
          source.Position += _endSymbol.Length;
          return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
        }
        
        source.Position++; 
      }

      throw new NotSupportedException();
    }
コード例 #7
0
        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;

            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return(null);
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return(source.CreateToken(this.OutputTerminal));
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return(context.CreateErrorToken("Unbalanced quoted string"));
        }
コード例 #8
0
        public override void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                var ch = source.PreviewChar;
                switch (ch)
                {
                case ' ':
                case '\t':
                case '\r':
                case '\n':
                case '\v':
                case '\u2085':
                case '\u2028':
                case '\u2029':
                    source.PreviewPosition++;
                    break;

                default:
                    //Check unicode class Zs
                    UnicodeCategory chCat = char.GetUnicodeCategory(ch);
                    if (chCat == UnicodeCategory.SpaceSeparator) //it is whitespace, continue moving
                    {
                        continue;                                //while loop
                    }
                    //Otherwize return
                    return;
                }
            }
        }
コード例 #9
0
        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;
            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return null;
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return source.CreateToken(this.OutputTerminal);
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced quoted string");
        }
コード例 #10
0
ファイル: FluidGrammar.cs プロジェクト: xiaolele1314/fluid
        public override void SkipWhitespace(ISourceStream source)
        {
            // This method is a copy of the base one with the additiong of the non-breaking space
            // c.f. https://github.com/sebastienros/fluid/issues/138

            const char NonBreakingSpace = (char)160;

            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                case ' ':
                case '\t':
                case NonBreakingSpace:
                    break;

                case '\r':
                case '\n':
                case '\v':
                    if (UsesNewLine)
                    {
                        return;                  //do not treat as whitespace if language is line-based
                    }
                    break;

                default:
                    return;
                }
                source.PreviewPosition++;
            }
        }
コード例 #11
0
ファイル: CommentTerminal.cs プロジェクト: kayateia/climoo
 private Token CompleteMatch(ParsingContext context, ISourceStream source) {
   //Find end symbol
   while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
     else 
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
     if (firstCharPos < 0) {
       source.PreviewPosition = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.PreviewPosition = firstCharPos;
     foreach (string endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol)) {
         //We found end symbol; eat end symbol only if it is not line comment.
         // For line comment, leave LF symbol there, it might be important to have a separate LF token
         if (!_isLineComment)
           source.PreviewPosition += endSymbol.Length;
         return source.CreateToken(this.OutputTerminal); 
       }//if
     }//foreach endSymbol
     source.PreviewPosition++; //move to the next char and try again    
   }//while
   return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }//method
コード例 #12
0
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            if (!ReadStartSymbol(source, details))
            {
                return(false);
            }

            bool   escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
            bool   ignoreCase    = IsSet(TermOptions.SpecialIgnoreCase);
            int    start         = source.Position;
            string startS        = details.ControlSymbol;
            string startS2       = startS + startS; //doubled start symbol
            //1. Find the string end
            // first get the position of the next line break; we are interested in it to detect malformed string,
            //  therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
            int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);

            while (!source.EOF())
            {
                int endPos = source.Text.IndexOf(startS, source.Position);
                //Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
                bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
                if (malformed)
                {
                    //Set source position for recovery: move to the next line if linebreak is not allowed.
                    if (nlPos > 0)
                    {
                        endPos = nlPos;
                    }
                    if (endPos > 0)
                    {
                        source.Position = endPos + 1;
                    }
                    details.Error = "Mal-formed  string literal - cannot find termination symbol.";
                    return(true);
                }

                //We found EndSymbol - check if it is escaped; if yes, skip it and continue search
                if (escapeEnabled && source.Text[endPos - 1] == EscapeChar)
                {
                    source.Position = endPos + startS.Length;
                    continue; //searching for end symbol
                }

                //Check if it is doubled end symbol
                source.Position = endPos;
                if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase))
                {
                    source.Position = endPos + startS.Length * 2;
                    continue;
                }//checking for doubled end symbol

                //Ok, this is normal endSymbol that terminates the string.
                // Advance source position and get out from the loop
                details.Body    = source.Text.Substring(start, endPos - start);
                source.Position = endPos + startS.Length;
                return(true); //if we come here it means we're done - we found string end.
            } //end of loop to find string end;
            return(false);
        }
コード例 #13
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;
              //Check starting symbol
              if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
              //Find end symbol
              source.Position += StartSymbol.Length;

              while(!source.EOF()) {
            int firstCharPos;
            if (EndSymbols.Count == 1)
              firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
            else
              firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
            if (firstCharPos < 0) {
              source.Position = source.Text.Length;
              if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
            }
            //We found a character that might start an end symbol; let's see if it is true.
            source.Position = firstCharPos;
            foreach (string endSymbol in EndSymbols)
              if (source.MatchSymbol(endSymbol, ignoreCase)) {
            //We found end symbol
            source.Position += endSymbol.Length;
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              }//if
            source.Position++; //move to the next char and try again
              }//while
              return null; //never happens
        }
コード例 #14
0
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            //remember start - it may be different from source.TokenStart, we may have skipped
            int start = source.Position;
            //Figure out digits set
            string digits     = GetDigits(details);
            bool   isDecimal  = !details.IsSet(ScanFlags.NonDecimal);
            bool   allowFloat = !IsSet(TermOptions.NumberIntOnly);

            while (!source.EOF())
            {
                char current = source.CurrentChar;
                //1. If it is a digit, just continue going
                if (digits.IndexOf(current) >= 0)
                {
                    source.Position++;
                    continue;
                }
                //2. Check if it is a dot
                if (current == DecimalSeparator && allowFloat)
                {
                    //If we had seen already a dot or exponent, don't accept this one;
                    //In python number literals (NumberAllowPointFloat) a point can be the first and last character,
                    //otherwise we accept dot only if it is followed by a digit
                    if (details.IsSet(ScanFlags.HasDotOrExp) || (digits.IndexOf(source.NextChar) < 0) && !IsSet(TermOptions.NumberAllowStartEndDot))
                    {
                        break; //from while loop
                    }
                    details.Flags |= ScanFlags.HasDot;
                    source.Position++;
                    continue;
                }
                //3. Only for decimals - check if it is (the first) exponent symbol
                if (allowFloat && isDecimal && (details.ControlSymbol == null) && (ExponentSymbols.IndexOf(current) >= 0))
                {
                    char next        = source.NextChar;
                    bool nextIsSign  = next == '-' || next == '+';
                    bool nextIsDigit = digits.IndexOf(next) >= 0;
                    if (!nextIsSign && !nextIsDigit)
                    {
                        break; //Exponent should be followed by either sign or digit
                    }
                    //ok, we've got real exponent
                    details.ControlSymbol = current.ToString(); //remember the exp char
                    details.Flags        |= ScanFlags.HasExp;
                    source.Position++;
                    if (nextIsSign)
                    {
                        source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
                    }
                    continue;
                }
                //4. It is something else (not digit, not dot or exponent) - we're done
                break; //from while loop
            }//while
            int end = source.Position;

            details.Body = source.Text.Substring(start, end - start);
            return(true);
        }
コード例 #15
0
ファイル: Grammar.cs プロジェクト: yangbodevp/irony
        /// <summary>Skips whitespace characters in the input stream. </summary>
        /// <remarks>Override this method if your language has non-standard whitespace characters.</remarks>
        /// <param name="source">Source stream.</param>
        public virtual void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                case ' ':
                case '\t':
                    break;

                case '\r':
                case '\n':
                case '\v':
                    if (UsesNewLine)
                    {
                        return;      //do not treat as whitespace if language is line-based
                    }
                    break;

                default:
                    return;
                } //switch
                source.PreviewPosition++;
            }     //while
        }         //method
コード例 #16
0
ファイル: StringLiteral.cs プロジェクト: eightrivers/SSharp
    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      bool isVerbatim = false;
      int start = source.Position;

      if (source.CurrentChar == '@')
      {
        isVerbatim = true;
        source.Position++;
        start++;
      }

      if (IsCurrentQuote(source))
      {
        source.Position++;
        start++;
      }
      else 
        return null;

      while (!source.EOF())
      {
        if (!isVerbatim)
        {
          if (source.CurrentChar == '\\')
          {
            //TODO: Escape processing
            source.Position += 2;
            continue;
          }
          else
            //Single line string ends incorrectly
            if (ParserData.LineTerminators.IndexOf(source.CurrentChar) >= 0)
              return null;
        }

        if (IsCurrentQuote(source)) break;
        
        source.Position++;
      }

      if (IsCurrentQuote(source))
        source.Position++;
      else
        return null;
      
      string lexeme = source.GetLexeme();
      string body = source.Text.Substring(start, source.Position - start - 1);
      //TODO: handle this in escape processing
      if (!isVerbatim)
        body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");

      TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);     
      return token;

      //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
    }
コード例 #17
0
        /// <summary>
        /// Tries to match an unquoted string from the source: this has been identified because the first character is
        /// not a single quote.
        /// </summary>
        /// <param name="context">
        /// The context in which the match is occuring.
        /// </param>
        /// <param name="source">
        /// The source to try to match.
        /// </param>
        /// <returns>
        /// Null if there is no match, an error token if the terminal is malformed and the appropriate token otherwise.
        /// </returns>
        private Token TryMatchUnquoted(ParsingContext context, ISourceStream source)
        {
            var openBraceCount = 0;
            var builder        = new StringBuilder();

            while (true)
            {
                if (source.EOF())
                {
                    return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
                }

                switch (source.PreviewChar)
                {
                case '\\':
                    ++source.PreviewPosition;
                    break;

                case '\'':
                    return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: unescaped quote encountered"));

                case '=':
                    if (openBraceCount <= 0)
                    {
                        return(this.CreateMemberNameToken(source, builder));
                    }

                    break;

                case ',':
                    if (openBraceCount <= 0)
                    {
                        return(this.CreateStringToken(context, source, builder));
                    }

                    break;

                case '{':
                    ++openBraceCount;
                    break;

                case '}':
                    if (openBraceCount-- <= 0)
                    {
                        return(this.CreateStringToken(context, source, builder));
                    }

                    break;
                }

                builder.Append(source.PreviewChar);
                ++source.PreviewPosition;
            }
        }
コード例 #18
0
        protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
        {
            int      start        = source.PreviewPosition;
            bool     allowEscapes = details.IsSet((short)IdOptions.AllowsEscapes);
            CharList outputChars  = new CharList();

            while (!source.EOF())
            {
                char current = source.PreviewChar;
                if (Grammar.IsWhitespaceOrDelimiter(current))
                {
                    break;
                }
                if (allowEscapes && current == this.EscapeChar)
                {
                    current = ReadUnicodeEscape(source, details);
                    //We  need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
                    //This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
                    // char is at position of last digit of escape sequence.
                    source.PreviewPosition--;
                    if (details.Error != null)
                    {
                        return(false);
                    }
                }
                //Check if current character is OK
                if (!CharOk(current, source.PreviewPosition == start))
                {
                    break;
                }
                //Check if we need to skip this char
#if NETSTANDARD
                UnicodeCategory currCat = CharUnicodeInfo.GetUnicodeCategory(current);
#else
                UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
#endif
                if (!this.CharsToRemoveCategories.Contains(currCat))
                {
                    outputChars.Add(current); //add it to output (identifier)
                }
                source.PreviewPosition++;
            }//while
            if (outputChars.Count == 0)
            {
                return(false);
            }
            //Convert collected chars to string
            details.Body = new string(outputChars.ToArray());
            if (!CheckCaseRestriction(details.Body))
            {
                return(false);
            }
            return(!string.IsNullOrEmpty(details.Body));
        }
コード例 #19
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
        {
            if (commentLevel == 0)
            {
                var line_breaks  = new char[] { '\n', '\r', '\v' };
                var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
                if (firstCharPos > 0)
                {
                    source.PreviewPosition = firstCharPos;
                }
                else
                {
                    source.PreviewPosition = source.Text.Length;
                }

                return(source.CreateToken(this.OutputTerminal));
            }

            while (!source.EOF())
            {
                string text    = source.Text.Substring(source.PreviewPosition);
                var    matches = Regex.Matches(text, @"\](=*)\]");
                foreach (Match match in matches)
                {
                    if (match.Groups[1].Value.Length == (int)commentLevel - 1)
                    {
                        source.PreviewPosition += match.Index + match.Length;

                        if (context.VsLineScanState.Value != 0)
                        {
                            //We are using line-mode and begin terminal was on previous line.
                            SourceLocation tokenStart = new SourceLocation();
                            tokenStart.Position = 0;

                            string lexeme = source.Text.Substring(0, source.PreviewPosition);

                            context.VsLineScanState.Value = 0;
                            return(new Token(this, tokenStart, lexeme, null));
                        }
                        else
                        {
                            return(source.CreateToken(this.OutputTerminal));
                        }
                    }
                }

                source.PreviewPosition++;
            }
            //The full match wasn't found, store the state for future parsing.
            //   context.VsLineScanState.TerminalIndex = this.MultilineIndex;
            context.VsLineScanState.TokenSubType = commentLevel;
            return(null);
        }
コード例 #20
0
        /// <summary>
        /// Tries to match a quoted string from the source: this has been identified because the first character is a
        /// single quote.
        /// </summary>
        /// <param name="context">
        /// The context in which the match is occuring.
        /// </param>
        /// <param name="source">
        /// The source to try to match.
        /// </param>
        /// <returns>
        /// Null if there is no match, an error token if the terminal is malformed and the appropriate token otherwise.
        /// </returns>
        private Token TryMatchQuoted(ParsingContext context, ISourceStream source)
        {
            ++source.PreviewPosition;
            var   isClosed = false;
            var   builder  = new StringBuilder();
            Token token    = null;

            while (token == null)
            {
                if (source.EOF())
                {
                    return(context.CreateErrorToken("Malformed MEMBERNAME or STRING: no terminal encountered"));
                }

                switch (source.PreviewChar)
                {
                case '\\':
                    ++source.PreviewPosition;
                    builder.Append(source.PreviewChar);
                    break;

                case '\'':
                    isClosed = true;
                    break;

                default:
                    if (!isClosed)
                    {
                        builder.Append(source.PreviewChar);
                        break;
                    }

                    if (char.IsWhiteSpace(source.PreviewChar))
                    {
                        break;
                    }

                    if (source.PreviewChar == '=')
                    {
                        token = this.CreateMemberNameToken(source, builder);
                        break;
                    }

                    token = this.CreateStringToken(context, source, builder);
                    break;
                }

                source.PreviewPosition++;
            }

            return(token);
        }
コード例 #21
0
ファイル: Program2.cs プロジェクト: WhiteGrouse/Walker
        private int?GetLength(ISourceStream source)
        {
            int length = 0;

            for (; !source.EOF() && char.IsDigit(source.PreviewChar); ++source.PreviewPosition)
            {
                length = length * 10 + int.Parse(source.PreviewChar.ToString());
            }
            if (length == 0 || source.PreviewPosition + length > source.Text.Length)
            {
                return(null);
            }
            return(length);
        }
コード例 #22
0
ファイル: CommentTerminal.cs プロジェクト: odasm/Fiesta_Utils
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;

            //Check starting symbol
            if (!source.MatchSymbol(StartSymbol, ignoreCase))
            {
                return(null);
            }
            //Find end symbol
            source.Position += StartSymbol.Length;

            while (!source.EOF())
            {
                int firstCharPos;
                if (EndSymbols.Count == 1)
                {
                    firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
                }
                else
                {
                    firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
                }
                if (firstCharPos < 0)
                {
                    source.Position = source.Text.Length;
                    if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
                    {
                        return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }
                    else
                    {
                        return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
                    }
                }
                //We found a character that might start an end symbol; let's see if it is true.
                source.Position = firstCharPos;
                foreach (string endSymbol in EndSymbols)
                {
                    if (source.MatchSymbol(endSymbol, ignoreCase))
                    {
                        //We found end symbol
                        source.Position += endSymbol.Length;
                        return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }//if
                }
                source.Position++; //move to the next char and try again
            } //while
            return(null); //never happens
        }     //method
コード例 #23
0
 private void MoveSourcePositionAfterTerminator(ISourceStream source)
 {
     while (!source.EOF())
     {
         while (source.PreviewChar != Terminator[0])
         {
             source.PreviewPosition++;
         }
         if (source.MatchSymbol(Terminator, !Grammar.CaseSensitive))
         {
             source.PreviewPosition += Terminator.Length;
             return;
         } //if
     }     //while
 }         //method
コード例 #24
0
        private Token CompleteMatch(ISourceStream source)
        {
            if (source.EOF())
            {
                return(null);
            }

            do
            {
                // Match NewLine
                var lookAhead = source.PreviewChar;
                if (LineTerminators.IndexOf(lookAhead) >= 0)
                {
                    source.PreviewPosition++;
                    // Treat \r\n as single NewLine
                    if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
                    {
                        source.PreviewPosition++;
                    }
                    break;
                }

                // Eat up whitespace
                if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
                {
                    source.PreviewPosition++;
                    continue;
                }

                // Fail on anything else
                return(null);
            }while (!source.EOF());

            // Create output token
            return(source.CreateToken(this.OutputTerminal));
        }
コード例 #25
0
 private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
 {
     if (commentLevel == 0)
     {
         char[] anyOf = new char[]
         {
             '\n',
             '\r',
             '\v'
         };
         int num = source.Text.IndexOfAny(anyOf, source.PreviewPosition);
         if (num > 0)
         {
             source.PreviewPosition = num;
         }
         else
         {
             source.PreviewPosition = source.Text.Length;
         }
         return(source.CreateToken(base.OutputTerminal));
     }
     while (!source.EOF())
     {
         foreach (Match match in Regex.Matches(source.Text.Substring(source.PreviewPosition), "\\](=*)\\]"))
         {
             if (match.Groups[1].Value.Length == (int)(commentLevel - 1))
             {
                 source.PreviewPosition += match.Index + match.Length;
                 Token result;
                 if (context.VsLineScanState.Value != 0)
                 {
                     SourceLocation location = default(SourceLocation);
                     location.Position = 0;
                     string text = source.Text.Substring(0, source.PreviewPosition);
                     context.VsLineScanState.Value = 0;
                     result = new Token(this, location, text, null);
                     return(result);
                 }
                 result = source.CreateToken(base.OutputTerminal);
                 return(result);
             }
         }
         int previewPosition = source.PreviewPosition;
         source.PreviewPosition = previewPosition + 1;
     }
     context.VsLineScanState.TokenSubType = commentLevel;
     return(null);
 }
コード例 #26
0
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            int      start        = source.Position;
            bool     allowEscapes = !details.IsSet(ScanFlags.DisableEscapes);
            CharList outputChars  = new CharList();

            while (!source.EOF())
            {
                char current = source.CurrentChar;
                if (_terminators.IndexOf(current) >= 0)
                {
                    break;
                }
                if (allowEscapes && current == this.EscapeChar)
                {
                    current = ReadUnicodeEscape(source, details);
                    //We  need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
                    //This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
                    // char is at position of last digit of escape sequence.
                    source.Position--;
                    if (details.HasError())
                    {
                        return(false);
                    }
                }
                //Check if current character is OK
                if (!CharOk(current, source.Position == start))
                {
                    break;
                }
                //Check if we need to skip this char
                UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
                if (!this.CharsToRemoveCategories.Contains(currCat))
                {
                    outputChars.Add(current); //add it to output (identifier)
                }
                source.Position++;
            }//while
            if (outputChars.Count == 0)
            {
                return(false);
            }
            //Convert collected chars to string
            details.Body = new string(outputChars.ToArray());
            return(!string.IsNullOrEmpty(details.Body));
        }
コード例 #27
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\n':
                    case '\r':
                    case ' ':
                    case '}':
                        if (source.PreviewPosition > source.Position)
                            return source.CreateToken(this.OutputTerminal);
                        return context.CreateErrorToken(Name + " was expected");
                }
                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced " + Name);
        }
コード例 #28
0
        protected override Token QuickParse(CompilerContext context, ISourceStream source)
        {
            if (AllFirstChars.IndexOf(source.CurrentChar) < 0)
            {
                return(null);
            }
            source.Position++;
            while (AllChars.IndexOf(source.CurrentChar) >= 0 && !source.EOF())
            {
                source.Position++;
            }
            //if it is not a terminator then cancel; we need to go through full algorithm
            if (_terminators.IndexOf(source.CurrentChar) < 0)
            {
                return(null);
            }
            string text = source.GetLexeme();

            return(Token.Create(this, context, source.TokenStart, text));
        }
コード例 #29
0
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            if (!IsValid(source.PreviewChar))
            {
                return(null);
            }
            source.PreviewPosition++;
            do
            {
                var current = source.PreviewChar;
                if (!IsValid(source.PreviewChar))
                {
                    break;
                }
                source.PreviewPosition++;
            }while (!source.EOF());
            var value = source.PreviewString();

            return(source.CreateToken(this.OutputTerminal, value));
        }
コード例 #30
0
ファイル: TypeNameTerminal.cs プロジェクト: ugurak/XamlStyler
        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                case '\n':
                case '\r':
                case ' ':
                case '}':
                    if (source.PreviewPosition > source.Position)
                    {
                        return(source.CreateToken(this.OutputTerminal));
                    }
                    return(context.CreateErrorToken(Name + " was expected"));
                }
                source.PreviewPosition++;
            }

            return(context.CreateErrorToken("Unbalanced " + Name));
        }
コード例 #31
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source)
        {
            //Find end symbol
            while (!source.EOF())
            {
                var firstCharPos = source.Text.IndexOf('{', source.PreviewPosition);
                if (firstCharPos < 0)
                {
                    source.PreviewPosition = source.Text.Length;
                    return(source.CreateToken(this.OutputTerminal));
                }
                //We found a character that might start an end symbol; let's see if it is true.
                source.PreviewPosition = firstCharPos;
                if (source.MatchSymbol("{{") || source.MatchSymbol("{%"))
                {
                    return(source.CreateToken(this.OutputTerminal));
                }
                source.PreviewPosition++; //move to the next char and try again
            }

            return(source.CreateToken(this.OutputTerminal));
        }
コード例 #32
0
ファイル: CommentTerminal.cs プロジェクト: singba/SSharp
        public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(_startSymbol, false))
            {
                return(null);
            }
            source.Position += _startSymbol.Length;

            while (!source.EOF())
            {
                int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

                if (firstCharPos < 0)
                {
                    source.Position = source.Text.Length;

                    if (_isLineComment)
                    {
                        return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }
                    else
                    {
                        return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
                    }
                }

                source.Position = firstCharPos;
                if (source.MatchSymbol(_endSymbol, false))
                {
                    source.Position += _endSymbol.Length;
                    return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
                }

                source.Position++;
            }

            throw new NotSupportedException();
        }
コード例 #33
0
 private Token CompleteMatch(CompilerContext context, ISourceStream source)
 {
     //Find end symbol
     while (!source.EOF())
     {
         int firstCharPos;
         if (EndSymbols.Count == 1)
         {
             firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
         }
         else
         {
             firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
         }
         if (firstCharPos < 0)
         {
             source.Position = source.Text.Length;
             return(null); //indicating error
         }
         //We found a character that might start an end symbol; let's see if it is true.
         source.Position = firstCharPos;
         foreach (string endSymbol in EndSymbols)
         {
             if (source.MatchSymbol(endSymbol, !OwnerGrammar.CaseSensitive))
             {
                 //We found end symbol; eat end symbol only if it is not line comment.
                 // For line comment, leave LF symbol there, it might be important to have a separate LF token
                 if (!_isLineComment)
                 {
                     source.Position += endSymbol.Length;
                 }
                 return(new Token(this, source.TokenStart, source.GetLexeme(), null));
             } //if
         }     //foreach endSymbol
         source.Position++; //move to the next char and try again
     }         //while
     return(null); //might happen if we found a start char of end symbol, but not the full endSymbol
 }             //method
コード例 #34
0
        private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
        {
            bool   escapeEnabled    = !details.IsSet((short)StringOptions.NoEscapes);
            int    start            = source.PreviewPosition;
            string endQuoteSymbol   = details.EndSymbol;
            string endQuoteDoubled  = endQuoteSymbol + endQuoteSymbol; //doubled quote symbol
            bool   lineBreakAllowed = details.IsSet((short)StringOptions.AllowsLineBreak);
            //1. Find the string end
            // first get the position of the next line break; we are interested in it to detect malformed string,
            //  therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
            int nlPos = lineBreakAllowed ? -1 : source.Text.IndexOf('\n', source.PreviewPosition);

            //fix by ashmind for EOF right after opening symbol
            while (true)
            {
                int endPos = source.Text.IndexOf(endQuoteSymbol, source.PreviewPosition);
                //Check for partial token in line-scanning mode
                if (endPos < 0 && details.PartialOk && lineBreakAllowed)
                {
                    ProcessPartialBody(source, details);
                    return(true);
                }
                //Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
                bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
                if (malformed)
                {
                    //Set source position for recovery: move to the next line if linebreak is not allowed.
                    if (nlPos > 0)
                    {
                        endPos = nlPos;
                    }
                    if (endPos > 0)
                    {
                        source.PreviewPosition = endPos + 1;
                    }
                    details.Error = Resources.ErrBadStrLiteral; //    "Mal-formed  string literal - cannot find termination symbol.";
                    return(true);                               //we did find start symbol, so it is definitely string, only malformed
                }//if malformed

                if (source.EOF())
                {
                    return(true);
                }

                //We found EndSymbol - check if it is escaped; if yes, skip it and continue search
                if (escapeEnabled && IsEndQuoteEscaped(source.Text, endPos))
                {
                    source.PreviewPosition = endPos + endQuoteSymbol.Length;
                    continue; //searching for end symbol
                }

                //Check if it is doubled end symbol
                source.PreviewPosition = endPos;
                if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && source.MatchSymbol(endQuoteDoubled, !CaseSensitive))
                {
                    source.PreviewPosition = endPos + endQuoteDoubled.Length;
                    continue;
                }//checking for doubled end symbol

                //Ok, this is normal endSymbol that terminates the string.
                // Advance source position and get out from the loop
                details.Body           = source.Text.Substring(start, endPos - start);
                source.PreviewPosition = endPos + endQuoteSymbol.Length;
                return(true); //if we come here it means we're done - we found string end.
            } //end of loop to find string end;
        }
コード例 #35
0
        private Token ReadToken()
        {
            if (_bufferedTokens.Count > 0)
            {
                Token tkn = _bufferedTokens[0];
                _bufferedTokens.RemoveAt(0);
                return(tkn);
            }
            //1. Skip whitespace. We don't need to check for EOF: at EOF we start getting 0-char, so we'll get out automatically
            while (_data.Grammar.WhitespaceChars.IndexOf(_source.CurrentChar) >= 0)
            {
                _source.Position++;
            }
            //That's the token start, calc location (line and column)
            SetTokenStartLocation();
            //Check for EOF
            if (_source.EOF())
            {
                return(Token.Create(Grammar.Eof, _context, _source.TokenStart, string.Empty, Grammar.Eof.Name));
            }
            //Find matching terminal
            // First, try terminals with explicit "first-char" prefixes, selected by current char in source
            TerminalList terms  = SelectTerminals(_source.CurrentChar);
            Token        result = MatchTerminals(terms);

            //If no token, try FallbackTerminals
            if (result == null && _data.FallbackTerminals.Count > 0)
            {
                result = MatchTerminals(_data.FallbackTerminals);
            }
            //If we don't have a token from registered terminals, try Grammar's method
            if (result == null)
            {
                result = _data.Grammar.TryMatch(_context, _source);
            }
            //Check if we have a multi-token; if yes, copy all but first child tokens from ChildNodes to _bufferedTokens,
            //  and set result to the first child token
            if (result != null && result.IsMultiToken())
            {
                foreach (Token tkn in result.ChildNodes)
                {
                    _bufferedTokens.Add(tkn);
                }
                result = _bufferedTokens[0];
                _bufferedTokens.RemoveAt(0);
            }
            //If we have normal token then return it
            if (result != null && !result.IsError())
            {
                //restore position to point after the result token
                _source.Position = _source.TokenStart.Position + result.Length;
                return(result);
            }
            //we have an error: either error token or no token at all
            if (result == null) //if no error result then create it
            {
                result = Grammar.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar);
            }
            Recover();
            return(result);
        }//method
コード例 #36
0
ファイル: SLangGrammar.cs プロジェクト: Rezura/LiveSplit
 public override void SkipWhitespace(ISourceStream source) {
   while (!source.EOF()) {
     var ch = source.PreviewChar;
     switch (ch) {
       case ' ':
       case '\t':
       case '\r':
       case '\n':
       case '\v':
       case '\u2085':
       case '\u2028':
       case '\u2029':
         source.PreviewPosition++;
         break;
       default:
         //Check unicode class Zs
         UnicodeCategory chCat = char.GetUnicodeCategory(ch);
         if (chCat == UnicodeCategory.SpaceSeparator) //it is whitespace, continue moving
           continue;//while loop 
         //Otherwize return
         return;
     }//switch
   }//while
 }
コード例 #37
0
ファイル: PowerShellGrammar.cs プロジェクト: mauve/Pash
        public override void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                int count = SkipSingleWhitespace(source);

                if (count == 0) return;

                else source.PreviewPosition += count;
            }
        }
コード例 #38
0
/*
 *  private static List<string> _firsts = new List<string>() { "'", "\"", "@" };
 */
        #endregion

        #region Init

        public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
        {
            bool isVerbatim = false;
            int  start      = source.Position;

            if (source.CurrentChar == '@')
            {
                isVerbatim = true;
                source.Position++;
                start++;
            }

            if (IsCurrentQuote(source))
            {
                source.Position++;
                start++;
            }
            else
            {
                return(null);
            }

            while (!source.EOF())
            {
                if (!isVerbatim)
                {
                    if (source.CurrentChar == '\\')
                    {
                        //TODO: Escape processing
                        source.Position += 2;
                        continue;
                    }
                    if (LRParser.LineTerminators.IndexOf(source.CurrentChar) >= 0)
                    {
                        return(null);
                    }
                }

                if (IsCurrentQuote(source))
                {
                    break;
                }

                source.Position++;
            }

            if (IsCurrentQuote(source))
            {
                source.Position++;
            }
            else
            {
                return(null);
            }

            string lexeme = source.GetLexeme();
            string body   = source.Text.Substring(start, source.Position - start - 1);

            //TODO: handle this in escape processing
            if (!isVerbatim)
            {
                body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");
            }

            TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);

            return(token);

            //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
        }
コード例 #39
0
        private Token MatchUnquoted(ParsingContext context, ISourceStream source)
        {
            if (source.PreviewChar == '{')
            {
                // Member names can't start with {
                if (this.IsMemberName)
                {
                    return null;
                }

                // Check for special {} at start of token indicating that this is a STRING token.
                if (source.NextPreviewChar != '}')
                {
                    return null;
                }

                source.PreviewPosition += 2;
            }

            var runningBraceTotal = 0;

            // This variable tracks the position of the last non whitespace (or significant whitespace).
            var lastNonWhitespacePosition = source.PreviewPosition;
            while (!source.EOF())
            {
                bool isWhiteSpace = false;
                switch (source.PreviewChar)
                {
                    case '{':
                        runningBraceTotal++;
                        break;

                    case '}':
                        if (--runningBraceTotal < 0)
                        {
                            return this.CreateToken(source, lastNonWhitespacePosition);
                        }

                        break;

                    case ',':
                        if (runningBraceTotal == 0)
                        {
                            return this.CreateToken(source, lastNonWhitespacePosition);
                        }

                        break;

                    case '=':
                        if (runningBraceTotal == 0)
                        {
                            // Equal sign. Only allowed after MemberNames.
                            return this.IsMemberName
                                ? this.CreateToken(source, lastNonWhitespacePosition)
                                : null;
                        }

                        break;

                    case '\\':
                        source.PreviewPosition++;
                        break;

                    default:
                        isWhiteSpace = Char.IsWhiteSpace(source.PreviewChar);
                        break;
                }

                source.PreviewPosition++;

                if (!isWhiteSpace)
                {
                    lastNonWhitespacePosition = source.PreviewPosition;
                }
            }

            return context.CreateErrorToken("Unterminated string terminal");
        }
コード例 #40
0
        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
        {
            if (commentLevel == 0)
            {
                var line_breaks = new char[] { '\n', '\r', '\v' };
                var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
                if (firstCharPos > 0)
                {
                    source.PreviewPosition = firstCharPos;
                }
                else
                {
                    source.PreviewPosition = source.Text.Length;
                }

                return source.CreateToken(this.OutputTerminal);
            }

            while (!source.EOF())
            {      
               string text = source.Text.Substring(source.PreviewPosition);
                var matches = Regex.Matches(text, @"\](=*)\]");
                foreach (Match match in matches)
                {
                    if (match.Groups[1].Value.Length == (int)commentLevel - 1)
                    {
                        source.PreviewPosition += match.Index + match.Length;

                        if (context.VsLineScanState.Value != 0)
                        {
                            SourceLocation tokenStart = new SourceLocation();
                            tokenStart.Position = 0;

                            string lexeme = source.Text.Substring(0, source.PreviewPosition);

                            context.VsLineScanState.Value = 0;
                            return new Token(this, tokenStart, lexeme, null);
                        }
                        else
                        {
                            return source.CreateToken(this.OutputTerminal);
                        }
                    }
                }

                source.PreviewPosition++;
            }
            
			context.VsLineScanState.TokenSubType = commentLevel;
            return null;
        }
コード例 #41
0
ファイル: NumberLiteral.cs プロジェクト: yangbodevp/irony
        }//method

        protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
        {
            //remember start - it may be different from source.TokenStart, we may have skipped prefix
            int  start   = source.PreviewPosition;
            char current = source.PreviewChar;

            if (IsSet(NumberOptions.AllowSign) && (current == '-' || current == '+'))
            {
                details.Sign = current.ToString();
                source.PreviewPosition++;
            }
            //Figure out digits set
            string digits      = GetDigits(details);
            bool   isDecimal   = !details.IsSet((short)(NumberOptions.Binary | NumberOptions.Octal | NumberOptions.Hex));
            bool   allowFloat  = !IsSet(NumberOptions.IntOnly);
            bool   foundDigits = false;

            while (!source.EOF())
            {
                current = source.PreviewChar;
                //1. If it is a digit, just continue going; the same for '_' if it is allowed
                if (digits.IndexOf(current) >= 0 || IsSet(NumberOptions.AllowUnderscore) && current == '_')
                {
                    source.PreviewPosition++;
                    foundDigits = true;
                    continue;
                }
                //2. Check if it is a dot in float number
                bool isDot = current == DecimalSeparator;
                if (allowFloat && isDot)
                {
                    //If we had seen already a dot or exponent, don't accept this one;
                    bool hasDotOrExp = details.IsSet((short)(NumberFlagsInternal.HasDot | NumberFlagsInternal.HasExp));
                    if (hasDotOrExp)
                    {
                        break;    //from while loop
                    }
                    //In python number literals (NumberAllowPointFloat) a point can be the first and last character,
                    //We accept dot only if it is followed by a digit
                    if (digits.IndexOf(source.NextPreviewChar) < 0 && !IsSet(NumberOptions.AllowStartEndDot))
                    {
                        break; //from while loop
                    }
                    details.Flags |= (int)NumberFlagsInternal.HasDot;
                    source.PreviewPosition++;
                    continue;
                }
                //3. Check if it is int number followed by dot or exp symbol
                bool isExpSymbol = (details.ExponentSymbol == null) && _exponentsTable.ContainsKey(current);
                if (!allowFloat && foundDigits && (isDot || isExpSymbol))
                {
                    //If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
                    if (IsSet(NumberOptions.NoDotAfterInt))
                    {
                        return(false);
                    }
                    //otherwise break, it is integer and we're done reading digits
                    break;
                }


                //4. Only for decimals - check if it is (the first) exponent symbol
                if (allowFloat && isDecimal && isExpSymbol)
                {
                    char next        = source.NextPreviewChar;
                    bool nextIsSign  = next == '-' || next == '+';
                    bool nextIsDigit = digits.IndexOf(next) >= 0;
                    if (!nextIsSign && !nextIsDigit)
                    {
                        break; //Exponent should be followed by either sign or digit
                    }
                    //ok, we've got real exponent
                    details.ExponentSymbol = current.ToString(); //remember the exp char
                    details.Flags         |= (int)NumberFlagsInternal.HasExp;
                    source.PreviewPosition++;
                    if (nextIsSign)
                    {
                        source.PreviewPosition++; //skip +/- explicitly so we don't have to deal with them on the next iteration
                    }
                    continue;
                }
                //4. It is something else (not digit, not dot or exponent) - we're done
                break; //from while loop
            }//while
            int end = source.PreviewPosition;

            if (!foundDigits)
            {
                return(false);
            }
            details.Body = source.Text.Substring(start, end - start);
            return(true);
        }
コード例 #42
0
ファイル: Grammar.cs プロジェクト: adrobyazko-softheme/PQL
 /// <summary>Skips whitespace characters in the input stream. </summary>
 /// <remarks>Override this method if your language has non-standard whitespace characters.</remarks>
 /// <param name="source">Source stream.</param>
 public virtual void SkipWhitespace(ISourceStream source)
 {
     while (!source.EOF()) {
     switch (source.PreviewChar) {
       case ' ':  case '\t':
     break;
       case '\r':   case '\n':  case '\v':
     if (UsesNewLine) return; //do not treat as whitespace if language is line-based
     break;
       default:
     return;
     }//switch
     source.PreviewPosition++;
       }//while
 }
コード例 #43
0
ファイル: DsvLiteral.cs プロジェクト: anukat2015/sones
 private void MoveSourcePositionAfterTerminator(ISourceStream source) {
   while(!source.EOF()) {
     while(source.PreviewChar != Terminator[0])
       source.PreviewPosition++;
     if(source.MatchSymbol(Terminator, !Grammar.CaseSensitive)) {
       source.PreviewPosition += Terminator.Length;
       return;
     }//if
   }//while
 }//method
コード例 #44
0
ファイル: BnfGrammar.cs プロジェクト: vf1/bnf2dfa
        public override void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\r':
                    case '\v':
                    case ' ':
                    case '\t':
                        break;

                    case '\n':
                        if (source.NextPreviewChar != ' ' &&
                            source.NextPreviewChar != '\t' &&
                            source.NextPreviewChar != '\n' &&
                            source.NextPreviewChar != '\r' &&
                            source.NextPreviewChar != '\v' &&
                            source.NextPreviewChar != ';')
                            return;
                        break;

                    default:
                        return;
                }

                source.PreviewPosition++;
            }
        }
コード例 #45
0
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            //remember start - it may be different from source.TokenStart, we may have skipped
              int start = source.Position;
              //Figure out digits set
              string digits = GetDigits(details);
              bool isDecimal = !details.IsSet(ScanFlags.NonDecimal);
              bool allowFloat = !IsSet(TermOptions.NumberIntOnly);

              while (!source.EOF()) {
            char current = source.CurrentChar;
            //1. If it is a digit, just continue going
            if (digits.IndexOf(current) >= 0) {
              source.Position++;
              continue;
            }
            //2. Check if it is a dot
            if (current == DecimalSeparator && allowFloat) {
              //If we had seen already a dot or exponent, don't accept this one;
              //In python number literals (NumberAllowPointFloat) a point can be the first and last character,
              //otherwise we accept dot only if it is followed by a digit
              if (details.IsSet(ScanFlags.HasDotOrExp) || (digits.IndexOf(source.NextChar) < 0) && !IsSet(TermOptions.NumberAllowStartEndDot))
            break; //from while loop
              details.Flags |= ScanFlags.HasDot;
              source.Position++;
              continue;
            }
            //3. Only for decimals - check if it is (the first) exponent symbol
            if (allowFloat && isDecimal && (details.ControlSymbol == null) && (ExponentSymbols.IndexOf(current) >= 0)) {
              char next = source.NextChar;
              bool nextIsSign = next == '-' || next == '+';
              bool nextIsDigit = digits.IndexOf(next) >= 0;
              if (!nextIsSign && !nextIsDigit)
            break;  //Exponent should be followed by either sign or digit
              //ok, we've got real exponent
              details.ControlSymbol = current.ToString(); //remember the exp char
              details.Flags |= ScanFlags.HasExp;
              source.Position++;
              if (nextIsSign)
            source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
              continue;
            }
            //4. It is something else (not digit, not dot or exponent) - we're done
            break; //from while loop
              }//while
              int end = source.Position;
              details.Body = source.Text.Substring(start, end - start);
              return true;
        }
コード例 #46
0
    protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details) {
      //remember start - it may be different from source.TokenStart, we may have skipped prefix
      int start = source.Position;
      char current = source.CurrentChar;
      if (current == '-' || current == '+') {
        details.Sign = current.ToString();
        source.Position++;
      }
      //Figure out digits set
      string digits = GetDigits(details);
      bool isDecimal = !details.IsSet((short) (NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex));
      bool allowFloat = !IsSet(NumberFlags.IntOnly);
      bool foundDigits = false;

      while (!source.EOF()) {
        current = source.CurrentChar;
        //1. If it is a digit, just continue going
        if (digits.IndexOf(current) >= 0) {
          source.Position++;
          foundDigits = true; 
          continue;
        }
        //2. Check if it is a dot in float number
        bool isDot = current == DecimalSeparator;
        if (allowFloat && isDot) {
          //If we had seen already a dot or exponent, don't accept this one;
          bool hasDotOrExp = details.IsSet((short) (NumberFlags.HasDot | NumberFlags.HasExp));
          if (hasDotOrExp) break; //from while loop
          //In python number literals (NumberAllowPointFloat) a point can be the first and last character,
          //We accept dot only if it is followed by a digit
          if (digits.IndexOf(source.NextChar) < 0 && !IsSet(NumberFlags.AllowStartEndDot))
            break; //from while loop
          details.Flags |= (int) NumberFlags.HasDot;
          source.Position++;
          continue;
        }
        //3. Check if it is int number followed by dot or exp symbol
        bool isExpSymbol = (details.ExponentSymbol == null) && ExponentSymbols.IndexOf(current) >= 0;
        if (!allowFloat && foundDigits && (isDot || isExpSymbol)) {
          //If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
          if (IsSet(NumberFlags.AvoidPartialFloat)) return false;  
          //otherwise break, it is integer and we're done reading digits
          break;
        }


        //4. Only for decimals - check if it is (the first) exponent symbol
        if (allowFloat && isDecimal && isExpSymbol) {
          char next = source.NextChar;
          bool nextIsSign = next == '-' || next == '+';
          bool nextIsDigit = digits.IndexOf(next) >= 0;
          if (!nextIsSign && !nextIsDigit)
            break;  //Exponent should be followed by either sign or digit
          //ok, we've got real exponent
          details.ExponentSymbol = current.ToString(); //remember the exp char
          details.Flags |= (int) NumberFlags.HasExp;
          source.Position++;
          if (nextIsSign)
            source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
          continue;
        }
        //4. It is something else (not digit, not dot or exponent) - we're done
        break; //from while loop
      }//while
      int end = source.Position;
      if (!foundDigits) 
        return false; 
      details.Body = source.Text.Substring(start, end - start);
      return true;
    }
コード例 #47
0
        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            if (!ReadStartSymbol(source, details)) return false;

              bool escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
              bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
              int start = source.Position;
              string startS = details.ControlSymbol;
              string startS2 = startS + startS; //doubled start symbol
              //1. Find the string end
              // first get the position of the next line break; we are interested in it to detect malformed string,
              //  therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
              int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);
              while (!source.EOF()) {
            int endPos = source.Text.IndexOf(startS, source.Position);
            //Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
            bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
            if (malformed) {
              //Set source position for recovery: move to the next line if linebreak is not allowed.
              if (nlPos > 0) endPos = nlPos;
              if (endPos > 0) source.Position = endPos + 1;
              details.Error = "Mal-formed  string literal - cannot find termination symbol.";
              return true;
            }

            //We found EndSymbol - check if it is escaped; if yes, skip it and continue search
            if (escapeEnabled && source.Text[endPos - 1] == EscapeChar) {
              source.Position = endPos + startS.Length;
              continue; //searching for end symbol
            }

            //Check if it is doubled end symbol
            source.Position = endPos;
            if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase)) {
              source.Position = endPos + startS.Length * 2;
              continue;
            }//checking for doubled end symbol

            //Ok, this is normal endSymbol that terminates the string.
            // Advance source position and get out from the loop
            details.Body = source.Text.Substring(start, endPos - start);
            source.Position = endPos + startS.Length;
            return true; //if we come here it means we're done - we found string end.
              }  //end of loop to find string end;
              return false;
        }