/// <summary> /// Pushes a new tokens collection /// </summary> /// <param name="_parent">The parent token hosting the collection</param> TokensCollection PushCollection(Token _parent) { TokensCollection newCollection = new TokensCollection(_parent); m_tokenCollections.Push(newCollection); return(newCollection); }
/// <summary> /// Retrieves the last token, that must be a name, and make it an alias instead /// </summary> void MakeLastTokenA(Token.TYPE _newType) { TokensCollection tokens = m_tokenCollections.Peek(); Token T = tokens.Last; if (T.m_type != Token.TYPE.NAME) { throw new Exception("Last token is not a name! Can't convert into an alias!"); } T.m_type = _newType; }
/// <summary> /// Obtiene todos los tokens de la cadena /// </summary> internal TokensCollection GetAllTokens() { TokensCollection tokens = new TokensCollection(); // Obtiene todos los tokens do { tokens.Add(GetNextToken()); }while (tokens[tokens.Count - 1].Type != Token.TokenType.EOF); // Devuelve la colección de tokens return(tokens); }
/// <summary> /// Adds a new token to the current collection /// </summary> /// <param name="T"></param> Token AddToken(Token.TYPE _type, object _value) { TokensCollection tokens = m_tokenCollections.Peek(); int streamIndex = m_stream.Index; if (_type == Token.TYPE.NAME && _value is Name) { streamIndex -= (_value as Name).ToString().Length; } Token T = new Token(streamIndex, _type, _value); tokens.Add(T); return(T); }
public Parser(string _text) { m_stream = new TextStream(_text + "\n"); Push(STATE.COLLECTION); m_root = PushCollection(null); Name name = null; char c; // Last read character in the stream while (!m_stream.EOT) { c = m_stream.Pop(); // Get current character if (IsSpaceNoEOL(c)) { continue; // Skip space } if (c == CHAR_COMMENT) { // Read till the end of line CommentParser.Read(m_stream); if (m_state.Peek() == STATE.NAME) { // In NAME state, a new line acts as a collection item separator Pop(); } continue; } if (c == CHAR_PREPROCESSOR) { // Read till the end of line and keep for further parsing... Token T = AddToken(Token.TYPE.PREPROCESSOR_DIRECTIVE, null); T.m_value = CommentParser.Read(m_stream); continue; } switch (m_state.Peek()) { ////////////////////////////////////////////////////////////////////////// // In NAME state, we expect: // • an alias assignment // • a child assignment // • a feature assignment // • a constructor // • a collection end // case STATE.NAME: if (c == CHAR_COLLECTION_ITEM_SEPARATOR || c == '\n') { // A new line acts as a collection item separator, go back to previous state Pop(); continue; } else if (c == CHAR_ALIAS_ASSIGNMENT || c == '⇒') { // Check if it's an assignment or a parent-child definition if (c == '⇒' || m_stream.Peek == CHAR_HIERARCHY) { // Parent-Child hierarchy marker if (c != '⇒') { c = m_stream.Pop(); // Consume character } AddToken(Token.TYPE.HIERARCHY_MARKER, null); Pop(); Push(STATE.CHILD); continue; } // Last parsed name was in fact an alias MakeLastTokenA(Token.TYPE.ALIAS); Pop(); Push(STATE.ALIAS); continue; } else if (c == CHAR_FEATURES_COLLECTION_MARKER) { // Parent-Child features marker AddToken(Token.TYPE.FEATURE_MARKER, null); Pop(); Push(STATE.CHILD); continue; } else if (c == CHAR_CONSTRUCTOR_START) { // Entering constructor definition Token T = AddToken(Token.TYPE.CONSTRUCTOR, null); T.m_value = PushCollection(T); Pop(); Push(STATE.CONSTRUCTOR); continue; } else if (c == CHAR_COLLECTION_END) { // Exiting collection Pop(); // Pop name Pop(); // Pop collection PopCollection(); Push(STATE.NAME); // Assume an entire collection is a single name token continue; } SyntaxError(m_stream, c, "a name"); break; ////////////////////////////////////////////////////////////////////////// // In VALUE state, we expect: // • A value separator // • The end of the constructor // case STATE.VALUE: if (c == CHAR_COLLECTION_ITEM_SEPARATOR || c == '\n') { // A new line acts as a separator, go back to previous state Pop(); continue; } else if (c == CHAR_CONSTRUCTOR_END) { // Exit the constructor Pop(); // Pop value Pop(); // Pop constructor PopCollection(); Push(STATE.NAME); // Assume a constructed name is a single name token continue; } SyntaxError(m_stream, c, "a constructor value"); break; ////////////////////////////////////////////////////////////////////////// // In collection state, we expect: // • a name // • a collection start // • a collection end // • a separator // case STATE.COLLECTION: if (c == CHAR_COLLECTION_ITEM_SEPARATOR || c == '\n') { continue; } if (c == CHAR_COLLECTION_START) { // Start a new collection Token T = AddToken(Token.TYPE.COLLECTION, null); T.m_value = PushCollection(T); Push(STATE.COLLECTION); continue; } else if (c == CHAR_COLLECTION_END) { // Exit the collection Pop(); // Pop collection PopCollection(); Push(STATE.NAME); // Assume an entire collection is a single name token continue; } else if (ReadName(c, m_stream, ref name)) { // We just read a name AddToken(Token.TYPE.NAME, name); Push(STATE.NAME); continue; } SyntaxError(m_stream, c, "collection"); break; ////////////////////////////////////////////////////////////////////////// // In CONSTRUCTOR state, we expect: // • A value // • An alias name // • The end of the constructor // case STATE.CONSTRUCTOR: if (c == '\n') { continue; } if (c == CHAR_CONSTRUCTOR_END) { // Exit the constructor Pop(); // Pop constructor PopCollection(); Push(STATE.NAME); // Assume a constructed name is a single name token continue; } else if (ReadName(c, m_stream, ref name)) { // We just read a name and it MUST be an alias! AddToken(Token.TYPE.NAME, name); Push(STATE.NAME); continue; } SyntaxError(m_stream, c, "a constructor"); break; ////////////////////////////////////////////////////////////////////////// // In CHILD state, we expect: // • a new NAME // • a new collection // case STATE.CHILD: if (c == '\n') { continue; } if (c == CHAR_COLLECTION_START) { // Start a new collection Pop(); Token T = AddToken(Token.TYPE.COLLECTION, null); T.m_value = PushCollection(T); Push(STATE.COLLECTION); continue; } else if (ReadName(c, m_stream, ref name)) { // We just read a name Pop(); AddToken(Token.TYPE.NAME, name); Push(STATE.NAME); continue; } SyntaxError(m_stream, c, "child"); break; ////////////////////////////////////////////////////////////////////////// // In ALIAS state, we expect: // • a new NAME // case STATE.ALIAS: if (c == '\n') { continue; } if (ReadName(c, m_stream, ref name)) { // We just read a name Pop(); AddToken(Token.TYPE.NAME, name); Push(STATE.NAME); continue; } SyntaxError(m_stream, c, "an alias"); break; } } // Verify we're back to the root collection if (m_state.Count != 1) { throw new Exception("Leaving parsing with " + (m_state.Count - 1) + " states remaining, not at root level!"); } if (m_state.Peek() != STATE.COLLECTION) { throw new Exception("Leaving parsing with a single state that is not the root COLLECTION!"); } }
internal ParserManager(Compiler compiler) { Compiler = compiler; Tokens = new TokensCollection(); _expressionEvaluator = new Evaluator.ExpressionConversorRpn(); }
public void TokensIteratorTest() { int tokensCount = 0; var tokensCollection = new TokensCollection(lexer); foreach (var lexeme in tokensCollection) { Assert.AreEqual(lexemes[tokensCount++], lexeme.ToString()); } Assert.AreEqual(lexemes.Length, tokensCount); }