/// <summary> /// Parses the current statement at the head of the token stream. It assumes the /// statement start token is still at the start of the stream. /// </summary> /// <returns> /// Filled NonTerminal object with the data of the statement parsed. /// </returns> private NonTerminal ParseNonTerminal() { List<IToken> tokensInStatement = new List<IToken>(); NonTerminal toReturn = new NonTerminal(NonTerminalType.LiteralText); // peek at the current token at the start of the stream. Do not pop it, it's thus not yet // removed from the stream. IToken currentToken = _tokenStream.Peek(); // check which statement we're looking at. switch((Token)currentToken.TokenID) { case Token.ColoredTextStartTag: toReturn = HandleColoredTextStart(tokensInStatement); break; case Token.ImageStartTag: toReturn = HandleImageStart(tokensInStatement); break; case Token.ListStartTag: toReturn = HandleListStart(tokensInStatement); break; case Token.QuotedTextStartTag: toReturn = HandleQuotedTextStart(tokensInStatement); break; case Token.SizedTextStartTag: toReturn = HandleSizedTextStart(tokensInStatement); break; case Token.URLStartTag: toReturn = HandleURLStart(tokensInStatement); break; case Token.BoldTextStartTag: case Token.BoldTextEndTag: case Token.CodeTextStartTag: case Token.CodeTextEndTag: case Token.ColoredTextEndTag: case Token.LF: case Token.EmailAddress: case Token.ImageURL: case Token.ItalicTextStartTag: case Token.ItalicTextEndTag: case Token.ListEndTag: case Token.ListItemEndTag: case Token.ListItemStartTag: case Token.OfftopicTextStartTag: case Token.OfftopicTextEndTag: case Token.QuotedTextEndTag: case Token.SizedTextEndTag: case Token.SmileyLaugh: case Token.SmileyAngry: case Token.SmileyRegular: case Token.SmileyWink: case Token.SmileyCool: case Token.SmileyTongue: case Token.SmileyConfused: case Token.SmileyShocked: case Token.SmileyDissapointed: case Token.SmileySad: case Token.SmileyEmbarrassed: case Token.StrikedTextStartTag: case Token.StrikedTextEndTag: case Token.Tab: case Token.UnderlinedTextStartTag: case Token.UnderlinedTextEndTag: case Token.URI: // single token tag toReturn = HandleSingleTokenTag(tokensInStatement); break; case Token.CR: // skip break; default: // error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); break; } return toReturn; }
/// <summary> /// Handles the URL start tokens and converts them into a nontermimal, if the syntax is correct. /// </summary> /// <param name="tokensInStatement">The tokens in statement.</param> /// <returns></returns> private NonTerminal HandleURLStart(List<IToken> tokensInStatement) { // first token has to be '[url', this routine gets called when the current token is [url if(!TestCurrentToken(Token.URLStartTag, "HandleURLStart", true)) { // failed. Exception is thrown by TestCurrentToken } // if is in the queue. Pop it. IToken currentToken = _tokenStream.Dequeue(); tokensInStatement.Add(currentToken); // we're inside a tag, therefor we can throw away whitespace. StripWhiteSpace(); NonTerminal toReturn = new NonTerminal(NonTerminalType.URLStart); // Next token is either tag end, or descriptionterminal. currentToken = (IToken)_tokenStream.Peek(); switch((Token)currentToken.TokenID) { case Token.TagCloseBracket: // valid argument. Pop it. tokensInStatement.Add(_tokenStream.Dequeue()); break; case Token.DescriptionTerminal: // description specified. Assignment and quoted string have to follow. tokensInStatement.Add(_tokenStream.Dequeue()); StripWhiteSpace(); // next token has to he assignment. if(!TestCurrentToken(Token.Assignment, "HandleURLStart", false)) { // not an assignment seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } // assignment seen. Pop it tokensInStatement.Add(_tokenStream.Dequeue()); StripWhiteSpace(); if(!TestCurrentToken(Token.QuotedString, "HandleURLStart", false)) { // not a quoted string seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } tokensInStatement.Add(_tokenStream.Dequeue()); // next token has to be a tagclosebracket StripWhiteSpace(); if(!TestCurrentToken(Token.TagCloseBracket, "HandleURLStart", false)) { // not an endbracket seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } tokensInStatement.Add(_tokenStream.Dequeue()); break; default: // error toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } // now strip whitespace and continue parsing. StripWhiteSpace(); // next token has to be an URI or ImageURL if((!TestCurrentToken(Token.URI, "HandleURLStart", false)) && (!TestCurrentToken(Token.ImageURL, "HandleURLStart", false))) { // not an URI / ImageURL seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } tokensInStatement.Add(_tokenStream.Dequeue()); StripWhiteSpace(); // now an urlend tag has to be there if(!TestCurrentToken(Token.URLEndTag, "HandleURLStart", false)) { // not an url end tag seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } tokensInStatement.Add(_tokenStream.Dequeue()); // done toReturn.Tokens.AddRange(tokensInStatement); return toReturn; }
/// <summary> /// Handles the Sized start tokens and converts them into a nontermimal, if the syntax is correct. /// </summary> /// <param name="tokensInStatement">The tokens in statement.</param> /// <returns></returns> private NonTerminal HandleSizedTextStart(List<IToken> tokensInStatement) { // first token has to be '[size', this routine gets called when the current token is [size if(!TestCurrentToken(Token.SizedTextStartTag, "HandleSizedTextStart", true)) { // failed. Exception is thrown by TestCurrentToken } // if is in the queue. Pop it. IToken currentToken = _tokenStream.Dequeue(); tokensInStatement.Add(currentToken); // we're inside a tag, therefor we can throw away whitespace. StripWhiteSpace(); NonTerminal toReturn = new NonTerminal(NonTerminalType.SizedTextStart); // Next token has to be valueTerminal. currentToken = (IToken)_tokenStream.Peek(); switch((Token)currentToken.TokenID) { case Token.ValueTerminal: // value specified. Assignment and (single) quoted string have to follow. tokensInStatement.Add(_tokenStream.Dequeue()); StripWhiteSpace(); // next token has to he assignment. if(!TestCurrentToken(Token.Assignment, "HandleSizedTextStart", false)) { // not an assignment seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } // assignment seen. Pop it tokensInStatement.Add(_tokenStream.Dequeue()); StripWhiteSpace(); // next token either is a quoted string or a single quoted string. currentToken = (IToken)_tokenStream.Peek(); switch((Token)currentToken.TokenID) { case Token.QuotedString: case Token.SingleQuotedNumericString: // valid break; default: // error toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } tokensInStatement.Add(_tokenStream.Dequeue()); // next token has to be a tagclosebracket StripWhiteSpace(); if(!TestCurrentToken(Token.TagCloseBracket, "HandleSizedTextStart", false)) { // not an endbracket seen. Error. toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } tokensInStatement.Add(_tokenStream.Dequeue()); break; default: // error toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement); // return now. return toReturn; } // done toReturn.Tokens.AddRange(tokensInStatement); // store this nonterminal onto its stack so a matching end tag can be linked with it. Stack<NonTerminal> stack = GetStartNTStack(NonTerminalType.SizedTextStart); stack.Push(toReturn); return toReturn; }
/// <summary> /// Handles the StatementStart single-token-statement-token StatementEnd statement /// </summary> /// <param name="tokensInStatement">Tokens already scanned</param> /// <returns>Filled NonTerminal with the tokens scanned. If there were no errors, this nonterminal will be /// of type EndIf. If there were errors, this nonterminal will be of type literaltext.</returns> private NonTerminal HandleSingleTokenTag(List<IToken> tokensInStatement) { NonTerminal toReturn = null; // store this nonterminal onto its stack so a matching end tag can be linked with it. Stack<NonTerminal> ntStack = null; IToken currentToken = (IToken)_tokenStream.Peek(); switch((Token)currentToken.TokenID) { case Token.BoldTextStartTag: toReturn = new NonTerminal(NonTerminalType.BoldTextStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.BoldTextEndTag: toReturn = new NonTerminal(NonTerminalType.BoldTextEnd); break; case Token.CodeTextStartTag: toReturn = new NonTerminal(NonTerminalType.CodeTextStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.CodeTextEndTag: toReturn = new NonTerminal(NonTerminalType.CodeTextEnd); break; case Token.ColoredTextEndTag: toReturn = new NonTerminal(NonTerminalType.ColoredTextEnd); break; case Token.LF: toReturn = new NonTerminal(NonTerminalType.CRLF); break; case Token.EmailAddress: toReturn = new NonTerminal(NonTerminalType.EmailAddress); break; case Token.ImageURL: toReturn = new NonTerminal(NonTerminalType.ImageURL); break; case Token.ItalicTextStartTag: toReturn = new NonTerminal(NonTerminalType.ItalicTextStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.ItalicTextEndTag: toReturn = new NonTerminal(NonTerminalType.ItalicTextEnd); break; case Token.ListEndTag: toReturn = new NonTerminal(NonTerminalType.ListEnd); break; case Token.ListItemEndTag: toReturn = new NonTerminal(NonTerminalType.ListItemEnd); break; case Token.ListItemStartTag: toReturn = new NonTerminal(NonTerminalType.ListItemStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.OfftopicTextStartTag: toReturn = new NonTerminal(NonTerminalType.OfftopicTextStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.OfftopicTextEndTag: toReturn = new NonTerminal(NonTerminalType.OfftopicTextEnd); break; case Token.QuotedTextEndTag: toReturn = new NonTerminal(NonTerminalType.QuotedTextEnd); break; case Token.SizedTextEndTag: toReturn = new NonTerminal(NonTerminalType.SizedTextEnd); break; case Token.SmileyLaugh: toReturn = new NonTerminal(NonTerminalType.SmileyLaugh); break; case Token.SmileyAngry: toReturn = new NonTerminal(NonTerminalType.SmileyAngry); break; case Token.SmileyRegular: toReturn = new NonTerminal(NonTerminalType.SmileyRegular); break; case Token.SmileyWink: toReturn = new NonTerminal(NonTerminalType.SmileyWink); break; case Token.SmileyCool: toReturn = new NonTerminal(NonTerminalType.SmileyCool); break; case Token.SmileyTongue: toReturn = new NonTerminal(NonTerminalType.SmileyTongue); break; case Token.SmileyConfused: toReturn = new NonTerminal(NonTerminalType.SmileyConfused); break; case Token.SmileyShocked: toReturn = new NonTerminal(NonTerminalType.SmileyShocked); break; case Token.SmileyDissapointed: toReturn = new NonTerminal(NonTerminalType.SmileyDissapointed); break; case Token.SmileySad: toReturn = new NonTerminal(NonTerminalType.SmileySad); break; case Token.SmileyEmbarrassed: toReturn = new NonTerminal(NonTerminalType.SmileyEmbarrassed); break; case Token.StrikedTextStartTag: toReturn = new NonTerminal(NonTerminalType.StrikedTextStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.StrikedTextEndTag: toReturn = new NonTerminal(NonTerminalType.StrikedTextEnd); break; case Token.Tab: toReturn = new NonTerminal(NonTerminalType.Tab); break; case Token.UnderlinedTextStartTag: toReturn = new NonTerminal(NonTerminalType.UnderlinedTextStart); ntStack = GetStartNTStack(toReturn.Type); break; case Token.UnderlinedTextEndTag: toReturn = new NonTerminal(NonTerminalType.UnderlinedTextEnd); break; case Token.URI: toReturn = new NonTerminal(NonTerminalType.URI); break; default: // error. This error is only caused by an internal parser error throw new ApplicationException("Internal parser error detected. 'HandleSingleStatement' expected a singel statement token but received token '" + ((IToken)_tokenStream.Peek()).TokenID.ToString() + "'."); } tokensInStatement.Add(_tokenStream.Dequeue()); toReturn.Tokens.AddRange(tokensInStatement); if(ntStack != null) { // push nonterminal as start nonterminal onto its stack so a related end nonterminal can relate to it. ntStack.Push(toReturn); } return toReturn; }
/// <summary> /// Converts the passed in set of popped tokens to one nonterminal of type LiteralText. This /// routine is used when an error is found in the stream so the currently popped tokens are /// apparantly not part of a statement, therefor should be converted to a literal string nonterminal, /// which will be transformed by the interpreter as a textstream. The error causing token is <i>not</i> included /// in the tokensPopped collection. /// </summary> /// <param name="tokensPopped">Set of tokens popped, EXCLUDING the error causing token, which will be used as one set /// of tokens of one nonterminal of type literal text</param> /// <returns>the literal text nonterminal with all the tokens passed in.</returns> private NonTerminal ConvertPoppedTokensToLiteralText(List<IToken> tokensPopped) { NonTerminal toReturn = new NonTerminal(NonTerminalType.LiteralText); IToken literalTextToken = new UBBToken(); literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString]; IToken currentToken = null; for(int i=0;i<tokensPopped.Count;i++) { currentToken = (IToken)tokensPopped[i]; literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText; } toReturn.Tokens.Add(literalTextToken); return toReturn; }
/// <summary> /// Parses the token stream into nonterminal objects. Uses simple LL(1) algo. /// Fills _nonTerminalStream; /// </summary> private void ParseTokens() { IToken literalTextToken = null; // parse the tokens bool eofFound = false; while((_tokenStream.Count > 0)&&!eofFound) { IToken currentToken = (IToken)_tokenStream.Peek(); int index = 0; switch((Token)currentToken.TokenID) { case Token.EOF: // done eofFound = true; break; case Token.BoldTextStartTag: case Token.BoldTextEndTag: case Token.CodeTextStartTag: case Token.CodeTextEndTag: case Token.ColoredTextStartTag: case Token.ColoredTextEndTag: case Token.LF: case Token.EmailAddress: case Token.ImageStartTag: case Token.ImageURL: case Token.ItalicTextStartTag: case Token.ItalicTextEndTag: case Token.ListEndTag: case Token.ListItemEndTag: case Token.ListItemStartTag: case Token.ListStartTag: case Token.OfftopicTextStartTag: case Token.OfftopicTextEndTag: case Token.QuotedTextStartTag: case Token.QuotedTextEndTag: case Token.SizedTextStartTag: case Token.SizedTextEndTag: case Token.SmileyLaugh: case Token.SmileyAngry: case Token.SmileyRegular: case Token.SmileyWink: case Token.SmileyCool: case Token.SmileyTongue: case Token.SmileyConfused: case Token.SmileyShocked: case Token.SmileyDissapointed: case Token.SmileySad: case Token.SmileyEmbarrassed: case Token.StrikedTextStartTag: case Token.StrikedTextEndTag: case Token.Tab: case Token.UnderlinedTextStartTag: case Token.UnderlinedTextEndTag: case Token.URLStartTag: case Token.URI: // Tag or otherwise nonterminal start. Handle statement. First create new nonterminal with current collected text, if available. if(literalTextToken!=null) { NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText); scannedText.Tokens.Add(literalTextToken); literalTextToken=null; _nonTerminalStream.Add(scannedText); index = (_nonTerminalStream.Count - 1); scannedText.CorrespondingEndNTIndex=index; } // token is still in the queue. NonTerminal parsedNonTerminal = ParseNonTerminal(); _nonTerminalStream.Add(parsedNonTerminal); index = (_nonTerminalStream.Count - 1); parsedNonTerminal.CorrespondingEndNTIndex = index; Stack<NonTerminal> ntStack = null; switch(parsedNonTerminal.Type) { case NonTerminalType.BoldTextEnd: ntStack = GetStartNTStack(NonTerminalType.BoldTextStart); break; case NonTerminalType.CodeTextEnd: ntStack = GetStartNTStack(NonTerminalType.CodeTextStart); break; case NonTerminalType.ColoredTextEnd: ntStack = GetStartNTStack(NonTerminalType.ColoredTextStart); break; case NonTerminalType.ItalicTextEnd: ntStack = GetStartNTStack(NonTerminalType.ItalicTextStart); break; case NonTerminalType.ListItemEnd: ntStack = GetStartNTStack(NonTerminalType.ListItemStart); break; case NonTerminalType.ListEnd: ntStack = GetStartNTStack(NonTerminalType.ListStart); break; case NonTerminalType.OfftopicTextEnd: ntStack = GetStartNTStack(NonTerminalType.OfftopicTextStart); break; case NonTerminalType.QuotedTextEnd: ntStack = GetStartNTStack(NonTerminalType.QuotedTextStart); break; case NonTerminalType.SizedTextEnd: ntStack = GetStartNTStack(NonTerminalType.SizedTextStart); break; case NonTerminalType.StrikedTextEnd: ntStack = GetStartNTStack(NonTerminalType.StrikedTextStart); break; case NonTerminalType.UnderlinedTextEnd: ntStack = GetStartNTStack(NonTerminalType.UnderlinedTextStart); break; } if((ntStack!=null) && (ntStack.Count > 0)) { NonTerminal startNT = ntStack.Pop(); startNT.CorrespondingEndNTIndex = index; } break; case Token.CR: // skip. We only use LF tokens for linefeeds. _tokenStream.Dequeue(); break; case Token.ImageEndTag: case Token.URLEndTag: default: // literal text or token which isn't defined properly. Push the complete text as // literal text into the current literal text token. currentToken = _tokenStream.Dequeue(); if(literalTextToken==null) { literalTextToken = new UBBToken(); literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString]; } literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText; break; } } // handle resulting text which is processed but is not transformed to a nonterminal yet. if(literalTextToken!=null) { NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText); scannedText.Tokens.Add(literalTextToken); literalTextToken=null; _nonTerminalStream.Add(scannedText); int index = (_nonTerminalStream.Count - 1); scannedText.CorrespondingEndNTIndex=index; } // change all nonterminals still on one of the stacks into literal text statements, since there are no matching endtokens found. foreach(Stack<NonTerminal> startNTStack in _startNTStacks.Values) { foreach(NonTerminal current in startNTStack) { current.Type = NonTerminalType.LiteralText; } } }