/// <summary> /// Performs syntax analysis against a sequence of terminals according to the <see cref="Grammar"/> used to create the <see cref="Parser"/>. /// </summary> /// <param name="terminalReader">Retrieves a sequence of <see cref="Terminal"/> objects.</param> /// <returns>If syntax analysis succeeds, returns the <see cref="Nonterminal"/> associated with <see cref="Grammar.StartNonterminal"/>. Otherwise, <value>null</value> is returned.</returns> public Nonterminal Parse(ITerminalReader terminalReader) { var stack = new ParserStack(); stack.Push(null, InitialState); var terminal = terminalReader.ReadTerminal(); while (terminal != null) { if (terminal.ElementType.Ignore) { terminal = terminalReader.ReadTerminal(); } else { var action = stack.Peek().State.GetAction(terminal.ElementType); LinguaTrace.TraceEvent(TraceEventType.Information, LinguaTraceId.ID_PARSE_ACTION, "{0}", action); if (action == null) { return null; } switch (action.ActionType) { case ParserActionTypes.Accept: { var reduce = (ParserActionAccept)action; var rule = reduce.Rule; var lhs = Reduce(stack, rule); return lhs; } case ParserActionTypes.Shift: { var shift = (ParserActionShift)action; stack.Push(terminal, shift.State); terminal = terminalReader.ReadTerminal(); } break; case ParserActionTypes.Reduce: { var reduce = (ParserActionReduce)action; var rule = reduce.Rule; var lhs = Reduce(stack, rule); // Push the LHS nonterminal on the stack. // stack.Push(lhs, stack.Peek().State.GetGoto(lhs.ElementType)); } break; default: throw new InvalidOperationException(string.Format("Unrecognized action type {0}.", action.ActionType)); } } } return null; }
public static bool IsCompatible(ATNState state, ParserStack stack) { var processResult = stack.Process(state); if (!processResult.isValid) { return(false); } return(!state.epsilonOnlyTransitions || state.Transitions.Any(transition => IsCompatible(transition.target, processResult.parserStack))); }
private void InitializeTables() { _stack = new ParserStack<State, TValue, TLocation>(); if (_tables == null) { lock (_tablesLock) { if (_tables == null) { Debug.Assert(typeof(TLocation).IsValueType); ParserTables tables = new ParserTables(); InitializeGeneratedTables(tables); #if DEBUG InitializeMetadata(tables); InitializeRulesMetadata(tables); #endif Thread.MemoryBarrier(); _tables = tables; } } } _states = _tables.States; _rules = _tables.Rules; _errorToken = _tables.ErrorToken; _eofToken = _tables.EofToken; }
private void ParseContents (SecurityElement e, bool restarted) { // // Iteratively collect stuff up until the next end-tag. // We've already seen the open-tag. // SecurityElementType lastType = SecurityElementType.Regular; ParserStack stack = new ParserStack(); ParserStackFrame firstFrame = new ParserStackFrame(); firstFrame.element = e; firstFrame.intag = false; stack.Push( firstFrame ); bool needToBreak = false; bool needToPop = false; int i; do { ParserStackFrame locFrame = stack.Peek(); for (i = _t.NextTokenType () ; i != -1 ; i = _t.NextTokenType ()) { switch (i) { case Tokenizer.cstr: { if (locFrame.intag) { if (locFrame.type == SecurityElementType.Comment) { String appendString; if (locFrame.sawEquals) { appendString = "=\"" + _t.GetStringToken() + "\""; locFrame.sawEquals = false; } else { appendString = " " + _t.GetStringToken(); } // Always set this directly since comments are not subjected // to the same restraints as other element types. The strings // are all escaped so this shouldn't be a problem. locFrame.element.Tag = locFrame.element.Tag + appendString; } else { // We're in a regular tag, so we've found an attribute/value pair. if (locFrame.strValue == null) { // Found attribute name, save it for later. locFrame.strValue = _t.GetStringToken (); } else { // Found attribute text, add the pair to the current element. if (!locFrame.sawEquals) throw new XmlSyntaxException( _t.LineNo ); locFrame.element.AddAttribute( locFrame.strValue, _t.GetStringToken() ); locFrame.strValue = null; } } } else { // We're not in a tag, so we've found text between tags. if (locFrame.element.Text == null) locFrame.element.Text = "" ; StringBuilder sb = new StringBuilder (locFrame.element.Text) ; // // Separate tokens with single spaces, collapsing whitespace // if (!locFrame.element.Text.Equals ("")) sb.Append (" ") ; sb.Append (_t.GetStringToken ()) ; locFrame.element.Text = sb.ToString (); } } break ; case Tokenizer.bra: locFrame.intag = true; i = _t.NextTokenType () ; if (i == Tokenizer.slash) { while (true) { // spin; don't care what's in here i = _t.NextTokenType(); if (i == Tokenizer.cstr) continue; else if (i == -1) throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_UnexpectedEndOfFile" )); else break; } if (i != Tokenizer.ket) { throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_ExpectedCloseBracket" )); } locFrame.intag = false; // Found the end of this element lastType = stack.Peek().type; stack.Pop(); needToBreak = true; } else if (i == Tokenizer.cstr) { // Found a child ParserStackFrame newFrame = new ParserStackFrame(); newFrame.element = new SecurityElement (_t.GetStringToken() ); if (locFrame.type != SecurityElementType.Regular) throw new XmlSyntaxException( _t.LineNo ); locFrame.element.AddChild (newFrame.element) ; stack.Push( newFrame ); needToBreak = true; } else if (i == Tokenizer.bang) { // Found a child that is a format node. Next up better be a cstr. ParserStackFrame newFrame = new ParserStackFrame(); newFrame.status = 1; do { i = _t.NextTokenType(); if (newFrame.status < 3) { if (i != Tokenizer.dash) throw new XmlSyntaxException( _t.LineNo ); else newFrame.status++; } else { if (i != Tokenizer.cstr) throw new XmlSyntaxException( _t.LineNo ); else break; } } while (true); newFrame.element = new SecurityElement (_t.GetStringToken()); newFrame.type = SecurityElementType.Comment; if (locFrame.type != SecurityElementType.Regular) throw new XmlSyntaxException( _t.LineNo ); locFrame.element.AddChild (newFrame.element) ; stack.Push( newFrame ); needToBreak = true; } else if (i == Tokenizer.quest) { // Found a child that is a format node. Next up better be a cstr. i = _t.NextTokenType(); if (i != Tokenizer.cstr) throw new XmlSyntaxException( _t.LineNo ); ParserStackFrame newFrame = new ParserStackFrame(); newFrame.element = new SecurityElement ( _t.GetStringToken()); newFrame.type = SecurityElementType.Format; if (locFrame.type != SecurityElementType.Regular) throw new XmlSyntaxException( _t.LineNo ); locFrame.element.AddChild (newFrame.element) ; newFrame.status = 1; stack.Push( newFrame ); needToBreak = true; } else { throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_ExpectedSlashOrString" )); } break ; case Tokenizer.equals: locFrame.sawEquals = true; break; case Tokenizer.ket: if (locFrame.intag) { locFrame.intag = false; continue; } else { throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_UnexpectedCloseBracket" )); } // not reachable case Tokenizer.slash: locFrame.element.Text = null; i = _t.NextTokenType (); if (i == Tokenizer.ket) { // Found the end of this element lastType = stack.Peek().type; stack.Pop(); needToBreak = true; } else { throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_ExpectedCloseBracket" )); } break; case Tokenizer.quest: if (locFrame.intag && locFrame.type == SecurityElementType.Format && locFrame.status == 1) { i = _t.NextTokenType (); if (i == Tokenizer.ket) { lastType = stack.Peek().type; stack.Pop(); needToBreak = true; } else { throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_ExpectedCloseBracket" )); } } else { throw new XmlSyntaxException (_t.LineNo); } break; case Tokenizer.dash: if (locFrame.intag && (locFrame.status > 0 && locFrame.status < 5) && locFrame.type == SecurityElementType.Comment) { locFrame.status++; if (locFrame.status == 5) { i = _t.NextTokenType (); if (i == Tokenizer.ket) { lastType = stack.Peek().type; stack.Pop(); needToBreak = true; } else { throw new XmlSyntaxException (_t.LineNo, caspol.manager.GetString( "XMLSyntax_ExpectedCloseBracket" )); } } } else { throw new XmlSyntaxException (_t.LineNo); } break; default: throw new XmlSyntaxException (_t.LineNo) ; } if (needToBreak) { needToBreak = false; needToPop = false; break; } else { needToPop = true; } } if (needToPop) { lastType = stack.Peek().type; stack.Pop(); } else if (i == -1 && stack.GetCount() != 1) { // This means that we still have items on the stack, but the end of our // stream has been reached. throw new XmlSyntaxException( _t.LineNo, caspol.manager.GetString( "XMLSyntax_UnexpectedEndOfFile" )); } } while (stack.GetCount() > 1); SecurityElement topElement = this.GetTopElement(); if (lastType == SecurityElementType.Format) { if (restarted) throw new XmlSyntaxException( _t.LineNo ); String format = topElement.Attribute( "encoding" ); if (format != null) { _t.ChangeFormat( System.Text.Encoding.GetEncoding( format ) ); } _ecurr = new SecurityElement( "junk" ); ParseContents( _ecurr, true ); } }
private void ParseContents (SecurityElement e) { // // Iteratively collect stuff up until the next end-tag. // We've already seen the open-tag. // ParserStack stack = new ParserStack(); ParserStackFrame firstFrame = new ParserStackFrame(); firstFrame.element = e; stack.Push( firstFrame ); bool needToBreak = false; bool needToPop = false; while (!stack.IsEmpty()) { ParserStackFrame locFrame = stack.Peek(); for (int i = _t.NextTokenType () ; i != -1 ; i = _t.NextTokenType ()) { switch (i) { case Tokenizer.cstr: { if (locFrame.intag) { // We're in a tag, so we've found an attribute/value pair. if (locFrame.strValue == null) { // Found attribute name, save it for later. locFrame.strValue = _t.GetStringToken (); } else { // Found attribute text, add the pair to the current element. locFrame.element.AddAttribute( locFrame.strValue, _t.GetStringToken () ); locFrame.strValue = null; } } else { // We're not in a tag, so we've found text between tags. if (locFrame.element.Text == null) locFrame.element.Text = "" ; StringBuilder sb = new StringBuilder (locFrame.element.Text) ; // // Separate tokens with single spaces, collapsing whitespace // if (!locFrame.element.Text.Equals ("")) sb.Append (" ") ; sb.Append (_t.GetStringToken ()) ; locFrame.element.Text = sb.ToString () ; } } break ; case Tokenizer.bra: locFrame.intag = true; i = _t.NextTokenType () ; if (i == Tokenizer.slash) { while ( (i = _t.NextTokenType()) == Tokenizer.cstr) ; // spin; don't care what's in here if (i != Tokenizer.ket) throw new XmlSyntaxException (_t.LineNo) ; // Found the end of this element stack.Pop(); needToBreak = true; } else if (i == Tokenizer.cstr) { // Found a child ParserStackFrame newFrame = new ParserStackFrame(); newFrame.element = new SecurityElement (_t.GetStringToken ()) ; locFrame.element.AddChild (newFrame.element) ; stack.Push( newFrame ); needToBreak = true; } else { throw new XmlSyntaxException (_t.LineNo) ; } break ; case Tokenizer.equals: break; case Tokenizer.ket: if (locFrame.intag) { locFrame.intag = false; continue; } else { throw new XmlSyntaxException (_t.LineNo); } // not reachable case Tokenizer.slash: locFrame.element.Text = null; i = _t.NextTokenType (); if (i == Tokenizer.ket) { // Found the end of this element stack.Pop(); needToBreak = true; } else { throw new XmlSyntaxException (_t.LineNo) ; } break; default: throw new XmlSyntaxException (_t.LineNo) ; } if (needToBreak) { needToBreak = false; needToPop = false; break; } else { needToPop = true; } } if (needToPop) { stack.Pop(); } } }
private void Suggest(ParserStack stack, ATNState state, int suggestionIndex) { var stackRes = stack.Process(state); if (!stackRes.isValid) { return; } foreach (var transition in state.Transitions) { if (transition.IsEpsilon && !_alreadyPassed.Contains(transition.target.stateNumber)) { _alreadyPassed.Add(transition.target.stateNumber); Suggest(stackRes.parserStack, transition.target, suggestionIndex); } else { switch (transition) { case AtomTransition atomTransition when(_allTokens.Count == 0 || _currentIndex >= _allTokens.Count) && _currentIndex == suggestionIndex: { if (ParserStack.IsCompatible(transition.target, stack)) { _suggestions.Add(atomTransition.label); } break; } case AtomTransition atomTransition: { var nextToken = _allTokens[_currentIndex]; if (_currentIndex == suggestionIndex && ParserStack.IsCompatible(transition.target, stack)) { _suggestions.Add(atomTransition.label); } else if (nextToken.Type == atomTransition.label) { if (_currentIndex + 1 > _allTokens.Count) { return; } _currentIndex++; Suggest(stackRes.parserStack, transition.target, suggestionIndex); } break; } case SetTransition setTransition when(_allTokens.Count == 0 || _currentIndex >= _allTokens.Count) && _currentIndex == suggestionIndex: { foreach (var tokenType in setTransition.Label.ToIntegerList()) { if (ParserStack.IsCompatible(transition.target, stack)) { _suggestions.Add(tokenType); } } break; } case SetTransition setTransition: { var nextToken = _allTokens[_currentIndex]; foreach (var tokenType in setTransition.Label.ToIntegerList()) { if (_currentIndex == suggestionIndex && ParserStack.IsCompatible(transition.target, stack)) { _suggestions.Add(tokenType); } else if (nextToken.Type == tokenType) { if (_currentIndex + 1 > _allTokens.Count) { return; } _currentIndex++; Suggest(stackRes.parserStack, transition.target, suggestionIndex); } } break; } default: throw new NotSupportedException(); } } } }
private static Nonterminal Reduce(ParserStack stack, RuleType rule) { // Create a language element array big enough to hold the LHS and RHS // arguments. // int parameterCount = 1 + rule.Rhs.Length; LanguageElement[] parameters = new LanguageElement[parameterCount]; // Create the LHS nonterminal. // Nonterminal lhs = rule.Lhs.CreateNonterminal(); parameters[0] = lhs; // Pop the RHS language elements off the stack. // for (int idx = 0; idx < rule.Rhs.Length; ++idx) { parameters[parameterCount - idx - 1] = stack.Pop().LanguageElement; } // Invoke the rule. // rule.Invoke(parameters); return lhs; }