Example #1
0
        public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
        {
            if (!source.MatchSymbol(_startSymbol, false))
            {
                return(null);
            }
            source.Position += _startSymbol.Length;

            while (!source.EOF())
            {
                int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

                if (firstCharPos < 0)
                {
                    source.Position = source.Text.Length;

                    return(_isLineComment
                   ? TokenAst.Create(this, context, source.TokenStart, source.GetLexeme())
                   : LRParser.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
                }

                source.Position = firstCharPos;
                if (source.MatchSymbol(_endSymbol, false))
                {
                    source.Position += _endSymbol.Length;
                    return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
                }

                source.Position++;
            }

            throw new NotSupportedException();
        }
Example #2
0
    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(_startSymbol, false)) return null;
      source.Position += _startSymbol.Length;

      while (!source.EOF())
      {
        int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

        if (firstCharPos < 0)
        {
          source.Position = source.Text.Length;

          if (_isLineComment)
            return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
          else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
        }
      
        source.Position = firstCharPos;
        if (source.MatchSymbol(_endSymbol, false))
        {
          source.Position += _endSymbol.Length;
          return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
        }
        
        source.Position++; 
      }

      throw new NotSupportedException();
    }
Example #3
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;

            //Check starting symbol
            if (!source.MatchSymbol(StartSymbol, ignoreCase))
            {
                return(null);
            }
            //Find end symbol
            source.Position += StartSymbol.Length;

            while (!source.EOF())
            {
                int firstCharPos;
                if (EndSymbols.Count == 1)
                {
                    firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
                }
                else
                {
                    firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
                }
                if (firstCharPos < 0)
                {
                    source.Position = source.Text.Length;
                    if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
                    {
                        return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }
                    else
                    {
                        return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
                    }
                }
                //We found a character that might start an end symbol; let's see if it is true.
                source.Position = firstCharPos;
                foreach (string endSymbol in EndSymbols)
                {
                    if (source.MatchSymbol(endSymbol, ignoreCase))
                    {
                        //We found end symbol
                        source.Position += endSymbol.Length;
                        return(Token.Create(this, context, source.TokenStart, source.GetLexeme()));
                    }//if
                }
                source.Position++; //move to the next char and try again
            } //while
            return(null); //never happens
        }     //method
Example #4
0
        protected virtual TokenAst CreateToken(CompilerContext context, ISourceStream source)
        {
            string   lexeme = source.GetLexeme();
            TokenAst token  = TokenAst.Create(this, context, source.TokenStart, lexeme, lexeme);

            return(token);
        }
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            Token result;

            if (context.ScannerState.Value != 0)
            {
                // we are continuing in line mode - restore internal env (none in this case)
                context.ScannerState.Value = 0;
            }
            else
            {
                //we are starting from scratch
                if (!BeginMatch(context, source))
                {
                    return(null);
                }
            }
            result = CompleteMatch(context, source);
            if (result != null)
            {
                return(result);
            }
            //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            if (_isLineComment)
            {
                return(new Token(this, source.TokenStart, source.GetLexeme(), null));
            }
            if (context.Mode == CompileMode.VsLineScan)
            {
                return(CreateIncompleteToken(context, source));
            }
            return(context.CreateErrorTokenAndReportError(source.TokenStart, string.Empty, "Unclosed comment block"));
        }
Example #6
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;
              //Check starting symbol
              if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
              //Find end symbol
              source.Position += StartSymbol.Length;

              while(!source.EOF()) {
            int firstCharPos;
            if (EndSymbols.Count == 1)
              firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
            else
              firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
            if (firstCharPos < 0) {
              source.Position = source.Text.Length;
              if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
            }
            //We found a character that might start an end symbol; let's see if it is true.
            source.Position = firstCharPos;
            foreach (string endSymbol in EndSymbols)
              if (source.MatchSymbol(endSymbol, ignoreCase)) {
            //We found end symbol
            source.Position += endSymbol.Length;
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              }//if
            source.Position++; //move to the next char and try again
              }//while
              return null; //never happens
        }
 public override Token TryMatch(CompilerContext context, ISourceStream source) {
   Match m = _expression.Match(source.Text, source.Position);
   if (!m.Success || m.Index != source.Position) 
     return null;
   source.Position += m.Length;
   string text = source.GetLexeme();
   return new Token(this, source.TokenStart, text, null);
 }
        protected virtual Token CreateToken(CompilerContext context, ISourceStream source, ScanDetails details)
        {
            string lexeme = source.GetLexeme();
            Token  token  = Token.Create(this, context, source.TokenStart, lexeme, details.Value);

            token.Details = details;
            return(token);
        }
        private Token CreateIncompleteToken(CompilerContext context, ISourceStream source)
        {
            source.Position = source.Text.Length;
            Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);

            result.Flags |= TokenFlags.IsIncomplete;
            context.ScannerState.TerminalIndex = this.MultilineIndex;
            return(result);
        }
 public override Token TryMatch(CompilerContext context, ISourceStream source)
 {
     Match m = _expression.Match(source.Text, source.Position);
       if (!m.Success)
     return null;
       source.Position += m.Length + 1;
       string text = source.GetLexeme();
       return Token.Create(this, context, source.TokenStart, text);
 }
Example #11
0
/*
    private static List<string> _firsts = new List<string>() { "'", "\"", "@" };
*/
    #endregion

    #region Init

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      bool isVerbatim = false;
      int start = source.Position;

      if (source.CurrentChar == '@')
      {
        isVerbatim = true;
        source.Position++;
        start++;
      }

      if (IsCurrentQuote(source))
      {
        source.Position++;
        start++;
      }
      else 
        return null;

      while (!source.EOF())
      {
        if (!isVerbatim)
        {
          if (source.CurrentChar == '\\')
          {
            //TODO: Escape processing
            source.Position += 2;
            continue;
          }
          if (LRParser.LineTerminators.IndexOf(source.CurrentChar) >= 0)
            return null;
        }

        if (IsCurrentQuote(source)) break;
        
        source.Position++;
      }

      if (IsCurrentQuote(source))
        source.Position++;
      else
        return null;
      
      string lexeme = source.GetLexeme();
      string body = source.Text.Substring(start, source.Position - start - 1);
      //TODO: handle this in escape processing
      if (!isVerbatim)
        body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");

      TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);     
      return token;

      //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
    }
 public override Token TryMatch(CompilerContext context, ISourceStream source) {
   char current = source.CurrentChar;
   if (!LineTerminators.Contains(current)) return null;
   //Treat \r\n as a single terminator
   bool doExtraShift = (current == '\r' && source.NextChar == '\n');
   source.Position++; //main shift
   if (doExtraShift)
     source.Position++;
   Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);
   return result;
 }
        protected virtual Token CreateToken(CompilerContext context, ISourceStream source, CompoundTokenDetails details)
        {
            string lexeme = source.GetLexeme();
            Token  token  = new Token(this, source.TokenStart, lexeme, details.Value);

            token.Details = details;
            if (details.IsPartial)
            {
                token.Flags |= TokenFlags.IsIncomplete;
            }
            return(token);
        }
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            Match m = _expression.Match(source.Text, source.Position);

            if (!m.Success)
            {
                return(null);
            }
            source.Position += m.Length + 1;
            string text = source.GetLexeme();

            return(Token.Create(this, context, source.TokenStart, text));
        }
Example #15
0
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            Match m = _expression.Match(source.Text, source.Position);

            if (!m.Success || m.Index != source.Position)
            {
                return(null);
            }
            source.Position += m.Length;
            string text = source.GetLexeme();

            return(new Token(this, source.TokenStart, text, null));
        }
Example #16
0
 public override Token TryMatch(CompilerContext context, ISourceStream source)
 {
     Token result;
       if (context.ScannerState.Value != 0) {
     // we are continuing in line mode - restore internal env (none in this case)
     context.ScannerState.Value = 0;
       } else {
     //we are starting from scratch
     if (!BeginMatch(context, source)) return null;
       }
       result = CompleteMatch(context, source);
       if (result != null) return result;
       //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
       if (_isLineComment)
     return new Token(this, source.TokenStart, source.GetLexeme(), null);
       if (context.Mode == CompileMode.VsLineScan)
     return CreateIncompleteToken(context, source);
       return context.CreateErrorTokenAndReportError(source.TokenStart, string.Empty, "Unclosed comment block");
 }
Example #17
0
        protected override Token QuickParse(CompilerContext context, ISourceStream source)
        {
            if (AllFirstChars.IndexOf(source.CurrentChar) < 0)
            {
                return(null);
            }
            source.Position++;
            while (AllChars.IndexOf(source.CurrentChar) >= 0 && !source.EOF())
            {
                source.Position++;
            }
            //if it is not a terminator then cancel; we need to go through full algorithm
            if (_terminators.IndexOf(source.CurrentChar) < 0)
            {
                return(null);
            }
            string text = source.GetLexeme();

            return(Token.Create(this, context, source.TokenStart, text));
        }
        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            char current = source.CurrentChar;

            if (!LineTerminators.Contains(current))
            {
                return(null);
            }
            //Treat \r\n as a single terminator
            bool doExtraShift = (current == '\r' && source.NextChar == '\n');

            source.Position++; //main shift
            if (doExtraShift)
            {
                source.Position++;
            }
            Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);

            return(result);
        }
 private Token CompleteMatch(CompilerContext context, ISourceStream source)
 {
     //Find end symbol
     while (!source.EOF())
     {
         int firstCharPos;
         if (EndSymbols.Count == 1)
         {
             firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
         }
         else
         {
             firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
         }
         if (firstCharPos < 0)
         {
             source.Position = source.Text.Length;
             return(null); //indicating error
         }
         //We found a character that might start an end symbol; let's see if it is true.
         source.Position = firstCharPos;
         foreach (string endSymbol in EndSymbols)
         {
             if (source.MatchSymbol(endSymbol, !OwnerGrammar.CaseSensitive))
             {
                 //We found end symbol; eat end symbol only if it is not line comment.
                 // For line comment, leave LF symbol there, it might be important to have a separate LF token
                 if (!_isLineComment)
                 {
                     source.Position += endSymbol.Length;
                 }
                 return(new Token(this, source.TokenStart, source.GetLexeme(), null));
             } //if
         }     //foreach endSymbol
         source.Position++; //move to the next char and try again
     }         //while
     return(null); //might happen if we found a start char of end symbol, but not the full endSymbol
 }             //method
Example #20
0
/*
 *  private static List<string> _firsts = new List<string>() { "'", "\"", "@" };
 */
        #endregion

        #region Init

        public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
        {
            bool isVerbatim = false;
            int  start      = source.Position;

            if (source.CurrentChar == '@')
            {
                isVerbatim = true;
                source.Position++;
                start++;
            }

            if (IsCurrentQuote(source))
            {
                source.Position++;
                start++;
            }
            else
            {
                return(null);
            }

            while (!source.EOF())
            {
                if (!isVerbatim)
                {
                    if (source.CurrentChar == '\\')
                    {
                        //TODO: Escape processing
                        source.Position += 2;
                        continue;
                    }
                    if (LRParser.LineTerminators.IndexOf(source.CurrentChar) >= 0)
                    {
                        return(null);
                    }
                }

                if (IsCurrentQuote(source))
                {
                    break;
                }

                source.Position++;
            }

            if (IsCurrentQuote(source))
            {
                source.Position++;
            }
            else
            {
                return(null);
            }

            string lexeme = source.GetLexeme();
            string body   = source.Text.Substring(start, source.Position - start - 1);

            //TODO: handle this in escape processing
            if (!isVerbatim)
            {
                body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");
            }

            TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);

            return(token);

            //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
        }
Example #21
0
 private Token CreateIncompleteToken(CompilerContext context, ISourceStream source)
 {
     source.Position = source.Text.Length;
       Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);
       result.Flags |= TokenFlags.IsIncomplete;
       context.ScannerState.TerminalIndex = this.MultilineIndex;
       return result;
 }
Example #22
0
 protected virtual TokenAst CreateToken(CompilerContext context, ISourceStream source)
 {
   string lexeme = source.GetLexeme();
   TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, lexeme);
   return token;
 }
Example #23
0
 private Token CompleteMatch(CompilerContext context, ISourceStream source)
 {
     //Find end symbol
       while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
     else
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
     if (firstCharPos < 0) {
       source.Position = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.Position = firstCharPos;
     foreach (String endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol, !Grammar.CaseSensitive)) {
     //We found end symbol; eat end symbol only if it is not line comment.
     // For line comment, leave LF symbol there, it might be important to have a separate LF token
     if (!_isLineComment)
       source.Position += endSymbol.Length;
     return Token.Create(this, context, source.TokenStart, source.GetLexeme());
       }//if
     }//foreach endSymbol
     source.Position++; //move to the next char and try again
       }//while
       return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }
Example #24
0
 private Token CreateIncompleteToken(CompilerContext context, ISourceStream source)
 {
     source.Position = source.Text.Length;
       Token result = Token.Create(this, context, source.TokenStart, source.GetLexeme());
       result.Flags |= AstNodeFlags.IsIncomplete;
       context.ScannerState.TokenKind = this.MultilineKind;
       return result;
 }