Exemple #1
0
        public bool Advance()
        {
            RegexTokeniser bestTokeniser;
            Match match;

            FindMatchingTokeniser(out bestTokeniser, out match);

            if (bestTokeniser != null)
            {
                Current = new Token(bestTokeniser.Type, match.Value, _currentPosition, _charNumber, _lineNumber);

                _currentPosition += match.Value.Length;

                int charactersAfterLineBreak;
                int linesInMatch;
                if (CurrentTokenSpansLineBreak(out charactersAfterLineBreak, out linesInMatch))
                {
                    _lineNumber += linesInMatch;
                    _charNumber += charactersAfterLineBreak;
                }
                else
                {
                    _charNumber += match.Value.Length;
                }

                return true;
            }

            Current = Token.Empty;

            return false;
        }
        public void TestTokenToString()
        {
            var t = new Token( '+' );
            string s = t.ToString();

            Assert.AreEqual( "+", s );
        }
        public Expression Parse(Parser parser, Expression lhs, Token current)
        {
            var thenExpression = parser.ParseNext();

            parser.Consume("COLON");

            var elseExpression = parser.ParseExpression(Precedence - 1);

            return new ConditionalExpr(lhs, thenExpression, elseExpression);
        }
        public Expression Parse(Parser parser, Expression lhs, Token current)
        {
            var functionName = ((IdentifierExpr) lhs);

            // we should already be on the leftparen token.
            var arguments = ParseArguments(parser).ToArray();

            parser.Consume("RIGHTPAREN");

            return new FunctionCallExpr(functionName, arguments);
        }
Exemple #5
0
        private Statement GetStatementFromToken(Token identifier, StatementTail tail)
        {
            var value = identifier != null ? identifier.Content : "";
            if (identifier != null)
            {
                switch (identifier.Type)
                {
                    case TokenType.StringLiteral:
                        return new StringLiteral(_host, value);
                    case TokenType.QuotedStringLiteral:
                        return new StringLiteral(_host, value);
                    case TokenType.StringLiteralPipe:
                        return new StringLiteralPipe(_host, value.Substring(1));
                    case TokenType.MultiLineStringLiteral:
                        return new StringLiteral(_host, value);
                }
            }

            return new Statement(_host, value, tail);
        }
Exemple #6
0
        static void Main()
        {
            File.Delete("J:\\Lexer12\\result.txt");
            FileStream file_key = new FileStream("J:\\Lexer12\\keys.txt", FileMode.Open, FileAccess.Read);
            StreamReader reader_key = new StreamReader(file_key);
            string k = reader_key.ReadToEnd();
            if (!File.Exists("J:\\Lexer12\\keywords.txt")) File.Create("J:\\Lexer12\\keywords.txt");
            if (!File.Exists("J:\\Lexer12\\identify.txt")) File.Create("J:\\Lexer12\\identify.txt");
            reader_key.Close();
            while (!String.IsNullOrEmpty(k))
            {
                string str = "";
                int i = 0;
                while (k[i] != ',')
                {
                    str = str + k[i];
                    i++;
                }
                if (!String.IsNullOrEmpty(k)) k = k.Substring(i + 1);
               // Console.WriteLine(Token.keys[0]);
               // Console.WriteLine(str);
                Token.keys.Add(str);
            }
            List<Token> ending = new List<Token>();
            FileStream file1 = new FileStream("J:\\Lexer12\\keywords.txt", FileMode.Open, FileAccess.Write);
            FileStream file2 = new FileStream("J:\\Lexer12\\identify.txt", FileMode.Open, FileAccess.Write);
             StreamWriter writer = new StreamWriter(file1);//
             StreamWriter writer1 = new StreamWriter(file2);//
            try
            {
                FileStream file = new FileStream("J:\\Lexer12\\text11.pas", FileMode.Open, FileAccess.Read);
                StreamReader reader = new StreamReader(file);
                string g = reader.ReadToEnd();
                string zap = g;
                string b="";
                bool l = control(ref g, ref b);
                zap = g;
                if (l)
                {
                    //Console.WriteLine(g);
                    //Console.WriteLine("---------------------------\r\n");
                    g = erase_space(g);

                    int i = 0;
                    //Console.WriteLine(g);
                    //Console.WriteLine("---------------------------\r\n");
                    while (!String.IsNullOrEmpty(g))
                    {
                        string f = get_str(ref g);
                        i++;
                        // writer.Write(f);
                        //writer.Write("\r\n");
                       Console.WriteLine(f);
                        bool log = true;
                        while (!String.IsNullOrEmpty(f))
                        {
                            Token r = new Token();
                            int a = 0;
                            string q = get_lex(ref f, ref log, ref a);
                            r.define(q, a, ref zap);
                            ending.Add(r);
                            // writer.Write(q);
                            // writer.Write("-");
                            //writer.Write(a);
                            // writer.Write("\r\n");
                        }
                        //Console.WriteLine(f);
                        //Console.WriteLine("---------------------------\r\n");
                    }
                    for (int j = 0; j < ending.Count; j++)
                    {
                        ending[j].conclude();
                    }
                    for (int j = 0; j < Token.keys.Count; j++)
                    {
                        writer.Write(j);
                        writer.Write(' ');
                        writer.WriteLine(Token.keys[j]);

                    }
                    //writer.WriteLine("-----------------------------------------------------");
                    for (int j = 0; j < Token.iden.Count; j++)
                    {
                        writer1.Write(j);
                        writer1.Write(' ');
                        writer1.WriteLine(Token.iden[j]);
                    }

                }
                else
                {
                    Console.WriteLine("Недопустимый(ые) символ(ы)");
                    Console.WriteLine(b);
                }
                reader.Close();
                writer.Close();
                writer1.Close();
            }
            catch
            {
                Console.WriteLine("Ошибка");
            }
            Console.ReadKey();
        }
 public Expression Parse(Parser parser, Token token)
 {
     return new ConstantExpr(Parse(token.Lexeme));
 }
    static void Main(string[] args)
    {
      Console.Write("\n  testing Semi");
      Console.Write("\n ==============");

      // Access Semi through interface and object factory.
      // That isolates client from any changes that may occur to Semi
      // as long as ITokenCollection doesn't change.

      ITokenCollection semi = Factory.create();

      string source = "../../semi.cs";
      if (!semi.open(source))
      {
        Console.Write("\n  Can't open {0}\n", source);
        return;
      }
      while (!semi.isDone())
      {
        semi.get();
        semi.show();
      }
      Console.Write("\n");

      Console.Write("\n  demonstrating semi operations");
      Console.Write("\n -------------------------------");

      ITokenCollection test = Factory.create();

      test.add("one").add("two").add("three");
      test.show();
      if (test.hasSequence("one", "three"))
        Console.Write("\n  semi has token \"one\" followed by token \"three\"");
      if (!test.hasSequence("foo", "two"))
        Console.Write("\n  semi does not have token \"foo\" followed by token \"two\"");
      if (!test.hasTerminator())
        Console.Write("\n  semi does not have terminator");

      Console.Write("\n  demonstrate changing semi with insert and add");
      test.insert(0, "#");
      test.add("\n");
      test.show();

      Console.Write("\n  demonstrate semi tests");
      if (test.hasTerminator())
        Console.Write("\n  semi has terminator");
      else
        Console.Write("\n  semi does not have terminator");

      int index;
      Token tok = "two";
      if (test.find(tok, out index))
        Console.Write("\n  found token \"{0}\" at position {1}", tok, index);
      else
        Console.Write("\n  did not find token \"{0}\"", tok);
      tok = "foo";
      if (test.find(tok, out index))
        Console.Write("\n  found token \"{0}\" at position {1}", tok, index);
      else
        Console.Write("\n  did not find token \"{0}\"", tok);

      tok = "one";
      Token tok2 = test.predecessor(tok);
      Console.Write("\n  predecessor of \"{0}\" is \"{1}\"", tok, tok2);
      tok = "bar";
      tok2 = test.predecessor(tok);
      Console.Write("\n  predecessor of \"{0}\" is \"{1}\"", tok, tok2);

      Console.Write("\n  indexing semi\n  ");
      for (int i = 0; i < test.size(); ++i)
        Console.Write("{0} ", test[i]);
      Console.Write("\n  using foreach:\n  ");
      foreach (var tk in test)
        Console.Write("{0} ", tk);

      Console.Write("\n\n");
    }
Exemple #9
0
 public Expression Parse(Parser parser, Token token)
 {
     var expression = parser.ParseNext();
     parser.Consume("RIGHTPAREN");
     return expression;
 }
Exemple #10
0
 public void Background(Token keyword, Token name)
 {
 }
        public void TestCreateToken()
        {
            var t = new Token( '+' );

            Assert.AreEqual( '+', t.Tag );
        }
Exemple #12
0
 public void ScenarioOutline(Token keyword, Token name)
 {
 }
Exemple #13
0
 public void Scenario(Token keyword, Token name)
 {
 }
Exemple #14
0
 public void PythonString(Token pyString)
 {
 }
Exemple #15
0
 public void Feature(Token keyword, Token name)
 {
 }
Exemple #16
0
 public void Examples(Token keyword, Token name)
 {
 }
Exemple #17
0
 public void Comment(Token content)
 {
 }
 public Expression Parse(Parser parser, Token current)
 {
     return new IdentifierExpr(current.Lexeme);
 }
Exemple #19
0
 public void Step(Token keyword, Token name, StepKind stepKind)
 {
 }
Exemple #20
0
 public void Tag(Token name)
 {
 }
Exemple #21
0
        public Token Scan( StringReader reader )
        {
            for ( ;; _peek = (char) reader.Read() )
            {
                if ( _peek == ' ' || _peek == '\t' )
                    continue;

                if ( _peek == '\n' )
                    Line += 1;
                else
                    break;
            }

            if ( Char.IsDigit( _peek ) )
            {
                var buf = new StringBuilder();
                do
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();
                } while( Char.IsDigit( _peek ) );

                if ( _peek == '.' )
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();

                    while( Char.IsDigit( _peek ) )
                    {
                        buf.Append( _peek );
                        _peek = (char)reader.Read();
                    }

                    CultureInfo ci = (CultureInfo)CultureInfo.CurrentCulture.Clone();
                    ci.NumberFormat.CurrencyDecimalSeparator = ".";

                    double d = double.Parse( buf.ToString(), NumberStyles.Any, ci );

                    return new Float( Tag.Float, d );
                }

                return new Num( Tag.Num, int.Parse( buf.ToString() ) );
            }

            if ( _peek == '.' )
            {
                var buf = new StringBuilder();

                buf.Append( _peek );
                _peek = (char)reader.Read();

                while ( Char.IsDigit( _peek ) )
                {
                    buf.Append( _peek );
                    _peek = (char)reader.Read();
                }

                CultureInfo ci = (CultureInfo)CultureInfo.CurrentCulture.Clone();
                ci.NumberFormat.CurrencyDecimalSeparator = ".";

                double d = double.Parse( buf.ToString(), NumberStyles.Any, ci );

                return new Float( Tag.Float, d );
            }

            if ( Char.IsLetter( _peek ) )
            {
                var buf = new StringBuilder();
                do
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();
                } while( Char.IsLetterOrDigit( _peek ) );
                string s = buf.ToString();
                var w = (Word) _words[s];
                if ( w != null )
                    return w;
                w = new Word( Tag.Id, s );
                _words.Add( s, w );
                return w;
            }

            if ( _peek == '/' )
            {
                _peek = (char) reader.Read();
                if ( _peek == '/' )
                {
                    _peek = (char) reader.Read();
                    var buf = new StringBuilder();
                    do
                    {
                        buf.Append( _peek );
                        _peek = (char) reader.Read();
                    } while( _peek != '\n' );

                    return new Comment( Tag.Comment, buf.ToString() );
                }

                if ( _peek == '*' )
                {
                    var buf = new StringBuilder();

                    _peek = (char) reader.Read();

                    while( true )
                    {
                        if ( _peek == '*' )
                        {
                            _peek = (char) reader.Read();
                            if ( _peek == '/' )
                            {
                                return new Comment( Tag.Comment, buf.ToString() );
                            }

                            buf.Append( '*' );
                        }
                        else
                        {
                            buf.Append( _peek );
                            _peek = (char) reader.Read();
                        }
                    }
                }
            }

            if ( _peek == '<' )
            {
                var buf = new StringBuilder();
                buf.Append( _peek );
                _peek = (char) reader.Read();

                if ( _peek == '=' )
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();

                    return new Word( Tag.LessOrEqual, buf.ToString() );
                }

                return new Word( Tag.Less, buf.ToString() );
            }

            if ( _peek == '>' )
            {
                var buf = new StringBuilder();
                buf.Append( _peek );
                _peek = (char) reader.Read();

                if ( _peek == '=' )
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();

                    return new Word( Tag.BetterOrEqual, buf.ToString() );
                }

                return new Word( Tag.Better, buf.ToString() );
            }

            if ( _peek == '=' )
            {
                var buf = new StringBuilder();
                buf.Append( _peek );
                _peek = (char) reader.Read();

                if ( _peek == '=' )
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();

                    return new Word( Tag.Equal, buf.ToString() );
                }
            }

            if ( _peek == '!' )
            {
                var buf = new StringBuilder();
                buf.Append( _peek );
                _peek = (char) reader.Read();

                if ( _peek == '=' )
                {
                    buf.Append( _peek );
                    _peek = (char) reader.Read();

                    return new Word( Tag.NotEqual, buf.ToString() );
                }
            }

            var t = new Token( _peek );
            _peek = ' ';
            return t;
        }
    //----< does semi contain specific token? >----------------------

    public bool contains(Token tok)
    {
      if (toks.Contains(tok))
        return true;
      return false;
    }
 public abstract Statement Parse(Parser parser, Token current);
    //----< add a token to the end of this semi-expression >---------

    public ITokenCollection add(Token token)
    {
      toks.Add(token);
      return this;
    }
Exemple #25
0
 public FakeScanner(Token [] tokens)
 {
     if (tokens == null)
         throw new ArgumentNullException();
     _tokens = tokens;
 }
Exemple #26
0
        public Token GetNextToken()
        {
            var token = new Token()
            {
                StartColumn = _currentColumn + 1, StartLine = _currentLine
            };
            TokenType lastErrorToken = TokenType.Error;

            if (_currentInputIndex >= _inputCode.Length)
            {
                token.TokenType = TokenType.EOF;
                return(token);
            }

            TokenType type        = TokenType.Intermediate;
            int       stateID     = 0;
            int       previousCol = 0;
            int       previousRow = 0;
            bool      startPosSet = false;

            _dfa.Begin();
            do
            {
                char currentChar = _currentInputIndex < _inputCode.Length ? _inputCode.ElementAt(_currentInputIndex) : '~'; // Insert EOF tag if past end of input
                if (!_dfa.IsCharacterInAlphabet(currentChar))
                {
                    _currentInputIndex++;
                    continue;
                }

                if (!startPosSet && !char.IsWhiteSpace(currentChar))
                {
                    token.StartColumn = _currentColumn + 1;
                    token.StartLine   = _currentLine;
                    startPosSet       = true;
                }

                previousCol = _currentColumn;
                previousRow = _currentLine;
                if (currentChar == '\n')
                {
                    _currentColumn = 0;
                    _currentLine++;
                }
                else
                {
                    _currentColumn++;
                }

                if (stateID == 1 && currentChar == '~')
                {
                    type = TokenType.EOF;
                    continue;
                }

                lastErrorToken = _dfa.GetErrorType(stateID);
                stateID        = _dfa.GetNextStateID(currentChar);
                type           = _dfa.GetStateType(stateID);

                token.Lexeme += currentChar;
                _currentInputIndex++;
            }while (type == TokenType.Intermediate);

            if (_dfa.IsBacktrackingState(stateID))
            {
                _currentInputIndex--;
                _currentColumn = previousCol;
                _currentLine   = previousRow;
                token.Lexeme   = token.Lexeme.Remove(token.Lexeme.Length - 1, 1);
                // undo col row position change
            }

            token.Lexeme = token.Lexeme.Trim().Replace("~", "");
            //token.StartLine = _currentLine;
            //token.StartColumn = _currentColumn;
            token.TokenType = type;

            if (token.TokenType == TokenType.Identifier)
            {
                var       keywordMapping = _dfa.GetKeywordMapping();
                TokenType keywordToken   = TokenType.Intermediate;
                if (keywordMapping.TryGetValue(token.Lexeme, out keywordToken))
                {
                    token.TokenType = keywordToken;
                }
            }

            if (token.TokenType == TokenType.Error)
            {
                token.TokenType = lastErrorToken;
            }

            return(token);
        }