/// <summary> /// Initializes new instance of Parser class. /// </summary> /// <param name="textReader">TextReader instance to read data from.</param> /// <param name="grammar">Grammar with parsing tables to parser input stream.</param> public Parser(TextReader textReader, Grammar grammar) { if (textReader == null) { throw new ArgumentNullException("textReader"); } if (grammar == null) { throw new ArgumentNullException("grammar"); } m_TextReader = textReader; m_BufferSize = MinimumBufferSize; m_Buffer = new Char[m_BufferSize + 1]; m_LineLength = Undefined; ReadBuffer(); m_InputTokens = new Token[MinimumInputTokenCount]; m_LRStack = new LRStackItem[MinimumLRStackSize]; m_Grammar = grammar; // Put grammar start symbol into LR parsing stack. m_LRState = m_Grammar.InitialLRState; var start = new LRStackItem { Token = { Symbol = m_Grammar.StartSymbol }, State = m_LRState }; m_LRStack[m_LRStackIndex] = start; m_CommentSymbols = new List<Regex>(); m_ReadOnlyCommentSymbols = new ReadOnlyCollection<Regex>(m_CommentSymbols); m_CommentEndSymbols = new Stack<Regex>(); m_ReductionCount = Undefined; // there are no reductions yet. }
private const int Undefined = -1; // Used for undefined int values. #endregion #region Constructors /// <summary> /// Initializes new instance of Parser class. /// </summary> /// <param name="textReader">TextReader instance to read data from.</param> /// <param name="grammar">Grammar with parsing tables to parser input stream.</param> public Parser(TextReader textReader, Grammar grammar) { if (textReader == null) { throw new ArgumentNullException("textReader"); } if (grammar == null) { throw new ArgumentNullException("grammar"); } m_textReader = textReader; m_bufferSize = MinimumBufferSize; m_buffer = new char[m_bufferSize + 1]; m_lineLength = Undefined; ReadBuffer(); m_inputTokens = new Token[MinimumInputTokenCount]; m_lrStack = new LRStackItem[MinimumLRStackSize]; m_grammar = grammar; // Put grammar start symbol into LR parsing stack. m_lrState = m_grammar.InitialLRState; LRStackItem start = new LRStackItem(); start.m_token.m_symbol = m_grammar.StartSymbol; start.m_state = m_lrState; m_lrStack[m_lrStackIndex] = start; m_reductionCount = Undefined; // there are no reductions yet. }
public void SetSourceCode(string sourceCode) { m_buffer = sourceCode; m_charIndex = 0; m_lineStart = 0; m_lineNumber = 1; m_lineLength = Undefined; // Put grammar start symbol into LR parsing stack. m_lrState = m_grammar.InitialLRState; m_lrStack[m_lrStackIndex] = new LRStackItem { m_token = { m_symbol = m_grammar.StartSymbol } }; m_reductionCount = Undefined; // there are no reductions yet. }
private TokenParseResult ParseToken() { LRStateAction stateAction = m_lrState.m_transitionVector[m_token.m_symbol.m_index]; if (stateAction != null) { //Work - shift or reduce if (m_reductionCount > 0) { int newIndex = m_lrStackIndex - m_reductionCount; m_lrStack[newIndex] = m_lrStack[m_lrStackIndex]; m_lrStackIndex = newIndex; } m_reductionCount = Undefined; switch (stateAction.Action) { case LRAction.Accept: m_reductionCount = 0; return(TokenParseResult.Accept); case LRAction.Shift: m_lrState = m_grammar.m_lrStateTable[stateAction.m_value]; LRStackItem nextToken = new LRStackItem(); nextToken.m_token = m_token; nextToken.m_state = m_lrState; if (m_lrStack.Length == ++m_lrStackIndex) { LRStackItem[] larger_m_lrStack = new LRStackItem[m_lrStack.Length + MinimumLRStackSize]; Array.Copy(m_lrStack, larger_m_lrStack, m_lrStack.Length); m_lrStack = larger_m_lrStack; } m_lrStack[m_lrStackIndex] = nextToken; return(TokenParseResult.Shift); case LRAction.Reduce: //Produce a reduction - remove as many tokens as members in the rule & push a nonterminal token int ruleIndex = stateAction.m_value; Rule currentRule = m_grammar.m_ruleTable[ruleIndex]; //======== Create Reduction LRStackItem head; TokenParseResult parseResult; LRState nextState; if (m_trimReductions && currentRule.m_hasOneNonTerminal) { //The current rule only consists of a single nonterminal and can be trimmed from the //parse tree. Usually we create a new Reduction, assign it to the Data property //of Head and push it on the stack. However, in this case, the Data property of the //Head will be assigned the Data property of the reduced token (i.e. the only one //on the stack). //In this case, to save code, the value popped of the stack is changed into the head. head = m_lrStack[m_lrStackIndex]; head.m_token.m_symbol = currentRule.m_nonTerminal; head.m_token.m_text = null; parseResult = TokenParseResult.ReduceEliminated; //========== Goto nextState = m_lrStack[m_lrStackIndex - 1].m_state; } else { //Build a Reduction head = new LRStackItem(); head.m_rule = currentRule; head.m_token.m_symbol = currentRule.m_nonTerminal; head.m_token.m_text = null; m_reductionCount = currentRule.m_symbols.Length; parseResult = TokenParseResult.ReduceNormal; //========== Goto nextState = m_lrStack[m_lrStackIndex - m_reductionCount].m_state; } //========= If nextAction is null here, then we have an Internal Table Error!!!! LRStateAction nextAction = nextState.m_transitionVector[currentRule.m_nonTerminal.m_index]; if (nextAction != null) { m_lrState = m_grammar.m_lrStateTable[nextAction.m_value]; head.m_state = m_lrState; if (parseResult == TokenParseResult.ReduceNormal) { if (m_lrStack.Length == ++m_lrStackIndex) { LRStackItem[] larger_m_lrStack = new LRStackItem[m_lrStack.Length + MinimumLRStackSize]; Array.Copy(m_lrStack, larger_m_lrStack, m_lrStack.Length); m_lrStack = larger_m_lrStack; } m_lrStack[m_lrStackIndex] = head; } else { m_lrStack[m_lrStackIndex] = head; } return(parseResult); } else { return(TokenParseResult.InternalError); } } } //=== Syntax Error! Fill Expected Tokens m_expectedTokens = new Symbol[m_lrState.ActionCount]; int length = 0; for (int i = 0; i < m_lrState.ActionCount; i++) { switch (m_lrState.GetAction(i).Symbol.SymbolType) { case SymbolType.Terminal: case SymbolType.End: m_expectedTokens[length++] = m_lrState.GetAction(i).Symbol; break; } } if (length < m_expectedTokens.Length) { Symbol[] newArray = new Symbol[length]; Array.Copy(m_expectedTokens, newArray, length); m_expectedTokens = newArray; } return(TokenParseResult.SyntaxError); }
private TokenParseResult ParseToken() { LRStateAction stateAction = m_lrState.m_transitionVector[m_token.m_symbol.m_index]; if (stateAction != null) { //Work - shift or reduce if (m_reductionCount > 0) { int newIndex = m_lrStackIndex - m_reductionCount; m_lrStack[newIndex] = m_lrStack[m_lrStackIndex]; m_lrStackIndex = newIndex; } m_reductionCount = Undefined; switch (stateAction.Action) { case LRAction.Accept: m_reductionCount = 0; return TokenParseResult.Accept; case LRAction.Shift: m_lrState = m_grammar.m_lrStateTable[stateAction.m_value]; LRStackItem nextToken = new LRStackItem(); nextToken.m_token = m_token; nextToken.m_state = m_lrState; if (m_lrStack.Length == ++m_lrStackIndex) { LRStackItem[] larger_m_lrStack = new LRStackItem[m_lrStack.Length + MinimumLRStackSize]; Array.Copy(m_lrStack, larger_m_lrStack, m_lrStack.Length); m_lrStack = larger_m_lrStack; } m_lrStack[m_lrStackIndex] = nextToken; return TokenParseResult.Shift; case LRAction.Reduce: //Produce a reduction - remove as many tokens as members in the rule & push a nonterminal token int ruleIndex = stateAction.m_value; Rule currentRule = m_grammar.m_ruleTable[ruleIndex]; //======== Create Reduction LRStackItem head; TokenParseResult parseResult; LRState nextState; if (m_trimReductions && currentRule.m_hasOneNonTerminal) { //The current rule only consists of a single nonterminal and can be trimmed from the //parse tree. Usually we create a new Reduction, assign it to the Data property //of Head and push it on the stack. However, in this case, the Data property of the //Head will be assigned the Data property of the reduced token (i.e. the only one //on the stack). //In this case, to save code, the value popped of the stack is changed into the head. head = m_lrStack[m_lrStackIndex]; head.m_token.m_symbol = currentRule.m_nonTerminal; head.m_token.m_text = null; parseResult = TokenParseResult.ReduceEliminated; //========== Goto nextState = m_lrStack[m_lrStackIndex - 1].m_state; } else { //Build a Reduction head = new LRStackItem(); head.m_rule = currentRule; head.m_token.m_symbol = currentRule.m_nonTerminal; head.m_token.m_text = null; m_reductionCount = currentRule.m_symbols.Length; parseResult = TokenParseResult.ReduceNormal; //========== Goto nextState = m_lrStack[m_lrStackIndex - m_reductionCount].m_state; } //========= If nextAction is null here, then we have an Internal Table Error!!!! LRStateAction nextAction = nextState.m_transitionVector[currentRule.m_nonTerminal.m_index]; if (nextAction != null) { m_lrState = m_grammar.m_lrStateTable[nextAction.m_value]; head.m_state = m_lrState; if (parseResult == TokenParseResult.ReduceNormal) { if (m_lrStack.Length == ++m_lrStackIndex) { LRStackItem[] larger_m_lrStack = new LRStackItem[m_lrStack.Length + MinimumLRStackSize]; Array.Copy(m_lrStack, larger_m_lrStack, m_lrStack.Length); m_lrStack = larger_m_lrStack; } m_lrStack[m_lrStackIndex] = head; } else { m_lrStack[m_lrStackIndex] = head; } return parseResult; } else { return TokenParseResult.InternalError; } } } //=== Syntax Error! Fill Expected Tokens m_expectedTokens = new Symbol[m_lrState.ActionCount]; int length = 0; for (int i = 0; i < m_lrState.ActionCount; i++) { switch (m_lrState.GetAction(i).Symbol.SymbolType) { case SymbolType.Terminal: case SymbolType.End: m_expectedTokens[length++] = m_lrState.GetAction(i).Symbol; break; } } if (length < m_expectedTokens.Length) { Symbol[] newArray = new Symbol[length]; Array.Copy(m_expectedTokens, newArray, length); m_expectedTokens = newArray; } return TokenParseResult.SyntaxError; }
private bool m_trimReductions = true; // Allowes to minimize reduction tree. #endregion Fields #region Constructors /// <summary> /// Initializes new instance of Parser class. /// </summary> /// <param name="textReader">TextReader instance to read data from.</param> /// <param name="grammar">Grammar with parsing tables to parser input stream.</param> public Parser(TextReader textReader, Grammar grammar) { if (textReader == null) { throw new ArgumentNullException("textReader"); } if (grammar == null) { throw new ArgumentNullException("grammar"); } m_textReader = textReader; m_bufferSize = MinimumBufferSize; m_buffer = new char[m_bufferSize + 1]; m_lineLength = Undefined; ReadBuffer(); m_inputTokens = new Token[MinimumInputTokenCount]; m_lrStack = new LRStackItem[MinimumLRStackSize]; m_grammar = grammar; // Put grammar start symbol into LR parsing stack. m_lrState = m_grammar.InitialLRState; LRStackItem start = new LRStackItem(); start.m_token.m_symbol = m_grammar.StartSymbol; start.m_state = m_lrState; m_lrStack[m_lrStackIndex] = start; m_reductionCount = Undefined; // there are no reductions yet. }
public void SetSourceCode(string sourceCode) { m_buffer = sourceCode; m_charIndex = 0; m_lineStart = 0; m_lineNumber = 1; m_lineLength = Undefined; // Put grammar start symbol into LR parsing stack. m_lrState = m_grammar.InitialLRState; m_lrStack[m_lrStackIndex] = new LRStackItem { m_token = { m_symbol = m_grammar.StartSymbol }}; m_reductionCount = Undefined; // there are no reductions yet. }
private void RsesizeLRStackIfNeed(int index) { if (index > m_lrStack.Length - 1) { LRStackItem[] tmp = new LRStackItem[m_lrStack.Length * 2]; m_lrStack.CopyTo(tmp, 0); m_lrStack = tmp; } }