Example #1
0
        /// <summary>
        /// Parses the current statement at the head of the token stream. It assumes the
        /// statement start token is still at the start of the stream.
        /// </summary>
        /// <returns>
        /// Filled NonTerminal object with the data of the statement parsed.
        /// </returns>
        private NonTerminal ParseNonTerminal()
        {
            List<IToken> tokensInStatement = new List<IToken>();

            NonTerminal toReturn = new NonTerminal(NonTerminalType.LiteralText);

            // peek at the current token at the start of the stream. Do not pop it, it's thus not yet
            // removed from the stream.
            IToken currentToken = _tokenStream.Peek();

            // check which statement we're looking at.
            switch((Token)currentToken.TokenID)
            {
                case Token.ColoredTextStartTag:
                    toReturn = HandleColoredTextStart(tokensInStatement);
                    break;
                case Token.ImageStartTag:
                    toReturn = HandleImageStart(tokensInStatement);
                    break;
                case Token.ListStartTag:
                    toReturn = HandleListStart(tokensInStatement);
                    break;
                case Token.QuotedTextStartTag:
                    toReturn = HandleQuotedTextStart(tokensInStatement);
                    break;
                case Token.SizedTextStartTag:
                    toReturn = HandleSizedTextStart(tokensInStatement);
                    break;
                case Token.URLStartTag:
                    toReturn = HandleURLStart(tokensInStatement);
                    break;
                case Token.BoldTextStartTag:
                case Token.BoldTextEndTag:
                case Token.CodeTextStartTag:
                case Token.CodeTextEndTag:
                case Token.ColoredTextEndTag:
                case Token.LF:
                case Token.EmailAddress:
                case Token.ImageURL:
                case Token.ItalicTextStartTag:
                case Token.ItalicTextEndTag:
                case Token.ListEndTag:
                case Token.ListItemEndTag:
                case Token.ListItemStartTag:
                case Token.OfftopicTextStartTag:
                case Token.OfftopicTextEndTag:
                case Token.QuotedTextEndTag:
                case Token.SizedTextEndTag:
                case Token.SmileyLaugh:
                case Token.SmileyAngry:
                case Token.SmileyRegular:
                case Token.SmileyWink:
                case Token.SmileyCool:
                case Token.SmileyTongue:
                case Token.SmileyConfused:
                case Token.SmileyShocked:
                case Token.SmileyDissapointed:
                case Token.SmileySad:
                case Token.SmileyEmbarrassed:
                case Token.StrikedTextStartTag:
                case Token.StrikedTextEndTag:
                case Token.Tab:
                case Token.UnderlinedTextStartTag:
                case Token.UnderlinedTextEndTag:
                case Token.URI:
                    // single token tag
                    toReturn = HandleSingleTokenTag(tokensInStatement);
                    break;
                case Token.CR:
                    // skip
                    break;
                default:
                    // error.
                    toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                    break;
            }

            return toReturn;
        }
Example #2
0
        /// <summary>
        /// Handles the URL start tokens and converts them into a nontermimal, if the syntax is correct. 
        /// </summary>
        /// <param name="tokensInStatement">The tokens in statement.</param>
        /// <returns></returns>
        private NonTerminal HandleURLStart(List<IToken> tokensInStatement)
        {
            // first token has to be '[url', this routine gets called when the current token is [url
            if(!TestCurrentToken(Token.URLStartTag, "HandleURLStart", true))
            {
                // failed. Exception is thrown by TestCurrentToken
            }

            // if is in the queue. Pop it.
            IToken currentToken = _tokenStream.Dequeue();
            tokensInStatement.Add(currentToken);

            // we're inside a tag, therefor we can throw away whitespace.
            StripWhiteSpace();

            NonTerminal toReturn = new NonTerminal(NonTerminalType.URLStart);

            // Next token is either tag end, or descriptionterminal.
            currentToken = (IToken)_tokenStream.Peek();
            switch((Token)currentToken.TokenID)
            {
                case Token.TagCloseBracket:
                    // valid argument. Pop it.
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    break;
                case Token.DescriptionTerminal:
                    // description specified. Assignment and quoted string have to follow.
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    StripWhiteSpace();
                    // next token has to he assignment.

                    if(!TestCurrentToken(Token.Assignment, "HandleURLStart", false))
                    {
                        // not an assignment seen. Error.
                        toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                        // return now.
                        return toReturn;
                    }
                    // assignment seen. Pop it
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    StripWhiteSpace();
                    if(!TestCurrentToken(Token.QuotedString, "HandleURLStart", false))
                    {
                        // not a quoted string seen. Error.
                        toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                        // return now.
                        return toReturn;
                    }
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    // next token has to be a tagclosebracket
                    StripWhiteSpace();
                    if(!TestCurrentToken(Token.TagCloseBracket, "HandleURLStart", false))
                    {
                        // not an endbracket seen. Error.
                        toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                        // return now.
                        return toReturn;
                    }
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    break;
                default:
                    // error
                    toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                    // return now.
                    return toReturn;
            }
            // now strip whitespace and continue parsing.
            StripWhiteSpace();
            // next token has to be an URI or ImageURL
            if((!TestCurrentToken(Token.URI, "HandleURLStart", false)) && (!TestCurrentToken(Token.ImageURL, "HandleURLStart", false)))
            {
                // not an URI / ImageURL seen. Error.
                toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                // return now.
                return toReturn;
            }
            tokensInStatement.Add(_tokenStream.Dequeue());

            StripWhiteSpace();
            // now an urlend tag has to be there
            if(!TestCurrentToken(Token.URLEndTag, "HandleURLStart", false))
            {
                // not an url end tag seen. Error.
                toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                // return now.
                return toReturn;
            }
            tokensInStatement.Add(_tokenStream.Dequeue());

            // done
            toReturn.Tokens.AddRange(tokensInStatement);
            return toReturn;
        }
Example #3
0
        /// <summary>
        /// Handles the Sized start tokens and converts them into a nontermimal, if the syntax is correct. 
        /// </summary>
        /// <param name="tokensInStatement">The tokens in statement.</param>
        /// <returns></returns>
        private NonTerminal HandleSizedTextStart(List<IToken> tokensInStatement)
        {
            // first token has to be '[size', this routine gets called when the current token is [size
            if(!TestCurrentToken(Token.SizedTextStartTag, "HandleSizedTextStart", true))
            {
                // failed. Exception is thrown by TestCurrentToken
            }

            // if is in the queue. Pop it.
            IToken currentToken = _tokenStream.Dequeue();
            tokensInStatement.Add(currentToken);

            // we're inside a tag, therefor we can throw away whitespace.
            StripWhiteSpace();

            NonTerminal toReturn = new NonTerminal(NonTerminalType.SizedTextStart);

            // Next token has to be valueTerminal.
            currentToken = (IToken)_tokenStream.Peek();
            switch((Token)currentToken.TokenID)
            {
                case Token.ValueTerminal:
                    // value specified. Assignment and (single) quoted string have to follow.
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    StripWhiteSpace();

                    // next token has to he assignment.
                    if(!TestCurrentToken(Token.Assignment, "HandleSizedTextStart", false))
                    {
                        // not an assignment seen. Error.
                        toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                        // return now.
                        return toReturn;
                    }
                    // assignment seen. Pop it
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    StripWhiteSpace();

                    // next token either is a quoted string or a single quoted string.
                    currentToken = (IToken)_tokenStream.Peek();
                    switch((Token)currentToken.TokenID)
                    {
                        case Token.QuotedString:
                        case Token.SingleQuotedNumericString:
                            // valid
                            break;
                        default:
                            // error
                            toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                            // return now.
                            return toReturn;
                    }
                    tokensInStatement.Add(_tokenStream.Dequeue());

                    // next token has to be a tagclosebracket
                    StripWhiteSpace();
                    if(!TestCurrentToken(Token.TagCloseBracket, "HandleSizedTextStart", false))
                    {
                        // not an endbracket seen. Error.
                        toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                        // return now.
                        return toReturn;
                    }
                    tokensInStatement.Add(_tokenStream.Dequeue());
                    break;
                default:
                    // error
                    toReturn = ConvertPoppedTokensToLiteralText(tokensInStatement);
                    // return now.
                    return toReturn;
            }
            // done
            toReturn.Tokens.AddRange(tokensInStatement);

            // store this nonterminal onto its stack so a matching end tag can be linked with it.
            Stack<NonTerminal> stack = GetStartNTStack(NonTerminalType.SizedTextStart);
            stack.Push(toReturn);

            return toReturn;
        }
Example #4
0
        /// <summary>
        /// Handles the StatementStart single-token-statement-token StatementEnd statement 
        /// </summary>
        /// <param name="tokensInStatement">Tokens already scanned</param>
        /// <returns>Filled NonTerminal with the tokens scanned. If there were no errors, this nonterminal will be 
        /// of type EndIf. If there were errors, this nonterminal will be of type literaltext.</returns>
        private NonTerminal HandleSingleTokenTag(List<IToken> tokensInStatement)
        {
            NonTerminal toReturn = null;

            // store this nonterminal onto its stack so a matching end tag can be linked with it.
            Stack<NonTerminal> ntStack = null;

            IToken currentToken = (IToken)_tokenStream.Peek();
            switch((Token)currentToken.TokenID)
            {
                case Token.BoldTextStartTag:
                    toReturn = new NonTerminal(NonTerminalType.BoldTextStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.BoldTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.BoldTextEnd);
                    break;
                case Token.CodeTextStartTag:
                    toReturn = new NonTerminal(NonTerminalType.CodeTextStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.CodeTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.CodeTextEnd);
                    break;
                case Token.ColoredTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.ColoredTextEnd);
                    break;
                case Token.LF:
                    toReturn = new NonTerminal(NonTerminalType.CRLF);
                    break;
                case Token.EmailAddress:
                    toReturn = new NonTerminal(NonTerminalType.EmailAddress);
                    break;
                case Token.ImageURL:
                    toReturn = new NonTerminal(NonTerminalType.ImageURL);
                    break;
                case Token.ItalicTextStartTag:
                    toReturn = new NonTerminal(NonTerminalType.ItalicTextStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.ItalicTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.ItalicTextEnd);
                    break;
                case Token.ListEndTag:
                    toReturn = new NonTerminal(NonTerminalType.ListEnd);
                    break;
                case Token.ListItemEndTag:
                    toReturn = new NonTerminal(NonTerminalType.ListItemEnd);
                    break;
                case Token.ListItemStartTag:
                    toReturn = new NonTerminal(NonTerminalType.ListItemStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.OfftopicTextStartTag:
                    toReturn = new NonTerminal(NonTerminalType.OfftopicTextStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.OfftopicTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.OfftopicTextEnd);
                    break;
                case Token.QuotedTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.QuotedTextEnd);
                    break;
                case Token.SizedTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.SizedTextEnd);
                    break;
                case Token.SmileyLaugh:
                    toReturn = new NonTerminal(NonTerminalType.SmileyLaugh);
                    break;
                case Token.SmileyAngry:
                    toReturn = new NonTerminal(NonTerminalType.SmileyAngry);
                    break;
                case Token.SmileyRegular:
                    toReturn = new NonTerminal(NonTerminalType.SmileyRegular);
                    break;
                case Token.SmileyWink:
                    toReturn = new NonTerminal(NonTerminalType.SmileyWink);
                    break;
                case Token.SmileyCool:
                    toReturn = new NonTerminal(NonTerminalType.SmileyCool);
                    break;
                case Token.SmileyTongue:
                    toReturn = new NonTerminal(NonTerminalType.SmileyTongue);
                    break;
                case Token.SmileyConfused:
                    toReturn = new NonTerminal(NonTerminalType.SmileyConfused);
                    break;
                case Token.SmileyShocked:
                    toReturn = new NonTerminal(NonTerminalType.SmileyShocked);
                    break;
                case Token.SmileyDissapointed:
                    toReturn = new NonTerminal(NonTerminalType.SmileyDissapointed);
                    break;
                case Token.SmileySad:
                    toReturn = new NonTerminal(NonTerminalType.SmileySad);
                    break;
                case Token.SmileyEmbarrassed:
                    toReturn = new NonTerminal(NonTerminalType.SmileyEmbarrassed);
                    break;
                case Token.StrikedTextStartTag:
                    toReturn = new NonTerminal(NonTerminalType.StrikedTextStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.StrikedTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.StrikedTextEnd);
                    break;
                case Token.Tab:
                    toReturn = new NonTerminal(NonTerminalType.Tab);
                    break;
                case Token.UnderlinedTextStartTag:
                    toReturn = new NonTerminal(NonTerminalType.UnderlinedTextStart);
                    ntStack = GetStartNTStack(toReturn.Type);
                    break;
                case Token.UnderlinedTextEndTag:
                    toReturn = new NonTerminal(NonTerminalType.UnderlinedTextEnd);
                    break;
                case Token.URI:
                    toReturn = new NonTerminal(NonTerminalType.URI);
                    break;
                default:
                    // error. This error is only caused by an internal parser error
                    throw new ApplicationException("Internal parser error detected. 'HandleSingleStatement' expected a singel statement token but received token '" + ((IToken)_tokenStream.Peek()).TokenID.ToString() + "'.");
            }

            tokensInStatement.Add(_tokenStream.Dequeue());
            toReturn.Tokens.AddRange(tokensInStatement);

            if(ntStack != null)
            {
                // push nonterminal as start nonterminal onto its stack so a related end nonterminal can relate to it.
                ntStack.Push(toReturn);
            }

            return toReturn;
        }
Example #5
0
        /// <summary>
        /// Converts the passed in set of popped tokens to one nonterminal of type LiteralText. This
        /// routine is used when an error is found in the stream so the currently popped tokens are
        /// apparantly not part of a statement, therefor should be converted to a literal string nonterminal,
        /// which will be transformed by the interpreter as a textstream. The error causing token is <i>not</i> included
        /// in the tokensPopped collection.
        /// </summary>
        /// <param name="tokensPopped">Set of tokens popped, EXCLUDING the error causing token, which will be used as one set
        /// of tokens of one nonterminal of type literal text</param>
        /// <returns>the literal text nonterminal with all the tokens passed in.</returns>
        private NonTerminal ConvertPoppedTokensToLiteralText(List<IToken> tokensPopped)
        {
            NonTerminal toReturn = new NonTerminal(NonTerminalType.LiteralText);

            IToken literalTextToken = new UBBToken();
            literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString];

            IToken currentToken = null;
            for(int i=0;i<tokensPopped.Count;i++)
            {
                currentToken = (IToken)tokensPopped[i];
                literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText;
            }

            toReturn.Tokens.Add(literalTextToken);
            return toReturn;
        }
Example #6
0
        /// <summary>
        /// Parses the token stream into nonterminal objects. Uses simple LL(1) algo.
        /// Fills _nonTerminalStream;
        /// </summary>
        private void ParseTokens()
        {
            IToken literalTextToken = null;

            // parse the tokens
            bool eofFound = false;
            while((_tokenStream.Count > 0)&&!eofFound)
            {
                IToken currentToken = (IToken)_tokenStream.Peek();

                int index = 0;
                switch((Token)currentToken.TokenID)
                {
                    case Token.EOF:
                        // done
                        eofFound = true;
                        break;
                    case Token.BoldTextStartTag:
                    case Token.BoldTextEndTag:
                    case Token.CodeTextStartTag:
                    case Token.CodeTextEndTag:
                    case Token.ColoredTextStartTag:
                    case Token.ColoredTextEndTag:
                    case Token.LF:
                    case Token.EmailAddress:
                    case Token.ImageStartTag:
                    case Token.ImageURL:
                    case Token.ItalicTextStartTag:
                    case Token.ItalicTextEndTag:
                    case Token.ListEndTag:
                    case Token.ListItemEndTag:
                    case Token.ListItemStartTag:
                    case Token.ListStartTag:
                    case Token.OfftopicTextStartTag:
                    case Token.OfftopicTextEndTag:
                    case Token.QuotedTextStartTag:
                    case Token.QuotedTextEndTag:
                    case Token.SizedTextStartTag:
                    case Token.SizedTextEndTag:
                    case Token.SmileyLaugh:
                    case Token.SmileyAngry:
                    case Token.SmileyRegular:
                    case Token.SmileyWink:
                    case Token.SmileyCool:
                    case Token.SmileyTongue:
                    case Token.SmileyConfused:
                    case Token.SmileyShocked:
                    case Token.SmileyDissapointed:
                    case Token.SmileySad:
                    case Token.SmileyEmbarrassed:
                    case Token.StrikedTextStartTag:
                    case Token.StrikedTextEndTag:
                    case Token.Tab:
                    case Token.UnderlinedTextStartTag:
                    case Token.UnderlinedTextEndTag:
                    case Token.URLStartTag:
                    case Token.URI:
                        // Tag or otherwise nonterminal start. Handle statement. First create new nonterminal with current collected text, if available.
                        if(literalTextToken!=null)
                        {
                            NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText);
                            scannedText.Tokens.Add(literalTextToken);
                            literalTextToken=null;

                            _nonTerminalStream.Add(scannedText);
                            index = (_nonTerminalStream.Count - 1);
                            scannedText.CorrespondingEndNTIndex=index;
                        }

                        // token is still in the queue.
                        NonTerminal parsedNonTerminal = ParseNonTerminal();
                        _nonTerminalStream.Add(parsedNonTerminal);
                        index = (_nonTerminalStream.Count - 1);
                        parsedNonTerminal.CorrespondingEndNTIndex = index;
                        Stack<NonTerminal> ntStack = null;
                        switch(parsedNonTerminal.Type)
                        {
                            case NonTerminalType.BoldTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.BoldTextStart);
                                break;
                            case NonTerminalType.CodeTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.CodeTextStart);
                                break;
                            case NonTerminalType.ColoredTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.ColoredTextStart);
                                break;
                            case NonTerminalType.ItalicTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.ItalicTextStart);
                                break;
                            case NonTerminalType.ListItemEnd:
                                ntStack = GetStartNTStack(NonTerminalType.ListItemStart);
                                break;
                            case NonTerminalType.ListEnd:
                                ntStack = GetStartNTStack(NonTerminalType.ListStart);
                                break;
                            case NonTerminalType.OfftopicTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.OfftopicTextStart);
                                break;
                            case NonTerminalType.QuotedTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.QuotedTextStart);
                                break;
                            case NonTerminalType.SizedTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.SizedTextStart);
                                break;
                            case NonTerminalType.StrikedTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.StrikedTextStart);
                                break;
                            case NonTerminalType.UnderlinedTextEnd:
                                ntStack = GetStartNTStack(NonTerminalType.UnderlinedTextStart);
                                break;
                        }
                        if((ntStack!=null) && (ntStack.Count > 0))
                        {
                            NonTerminal startNT = ntStack.Pop();
                            startNT.CorrespondingEndNTIndex = index;
                        }

                        break;
                    case Token.CR:
                        // skip. We only use LF tokens for linefeeds.
                        _tokenStream.Dequeue();
                        break;
                    case Token.ImageEndTag:
                    case Token.URLEndTag:
                    default:
                        // literal text or token which isn't defined properly. Push the complete text as
                        // literal text into the current literal text token.
                        currentToken = _tokenStream.Dequeue();
                        if(literalTextToken==null)
                        {
                            literalTextToken = new UBBToken();
                            literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString];
                        }
                        literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText;
                        break;
                }
            }

            // handle resulting text which is processed but is not transformed to a nonterminal yet.
            if(literalTextToken!=null)
            {
                NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText);
                scannedText.Tokens.Add(literalTextToken);
                literalTextToken=null;

                _nonTerminalStream.Add(scannedText);
                int index = (_nonTerminalStream.Count - 1);
                scannedText.CorrespondingEndNTIndex=index;
            }

            // change all nonterminals still on one of the stacks into literal text statements, since there are no matching endtokens found.
            foreach(Stack<NonTerminal> startNTStack in _startNTStacks.Values)
            {
                foreach(NonTerminal current in startNTStack)
                {
                    current.Type = NonTerminalType.LiteralText;
                }
            }
        }