/// <summary> /// Creates a new token object using this definition and prefills it. /// </summary> /// <returns>An TDLToken token</returns> public IToken CreateTokenFromDefinition() { UBBToken toReturn = new UBBToken(); toReturn.RelatedTokenDefinition = this; return(toReturn); }
/// <summary> /// Creates a new token object using this definition and prefills it. /// </summary> /// <returns>An TDLToken token</returns> public IToken CreateTokenFromDefinition() { UBBToken toReturn = new UBBToken(); toReturn.RelatedTokenDefinition = this; return toReturn; }
/// <summary> /// Converts the passed in set of popped tokens to one nonterminal of type LiteralText. This /// routine is used when an error is found in the stream so the currently popped tokens are /// apparantly not part of a statement, therefor should be converted to a literal string nonterminal, /// which will be transformed by the interpreter as a textstream. The error causing token is <i>not</i> included /// in the tokensPopped collection. /// </summary> /// <param name="tokensPopped">Set of tokens popped, EXCLUDING the error causing token, which will be used as one set /// of tokens of one nonterminal of type literal text</param> /// <returns>the literal text nonterminal with all the tokens passed in.</returns> private NonTerminal ConvertPoppedTokensToLiteralText(List<IToken> tokensPopped) { NonTerminal toReturn = new NonTerminal(NonTerminalType.LiteralText); IToken literalTextToken = new UBBToken(); literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString]; IToken currentToken = null; for(int i=0;i<tokensPopped.Count;i++) { currentToken = (IToken)tokensPopped[i]; literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText; } toReturn.Tokens.Add(literalTextToken); return toReturn; }
/// <summary> /// Parses the token stream into nonterminal objects. Uses simple LL(1) algo. /// Fills _nonTerminalStream; /// </summary> private void ParseTokens() { IToken literalTextToken = null; // parse the tokens bool eofFound = false; while((_tokenStream.Count > 0)&&!eofFound) { IToken currentToken = (IToken)_tokenStream.Peek(); int index = 0; switch((Token)currentToken.TokenID) { case Token.EOF: // done eofFound = true; break; case Token.BoldTextStartTag: case Token.BoldTextEndTag: case Token.CodeTextStartTag: case Token.CodeTextEndTag: case Token.ColoredTextStartTag: case Token.ColoredTextEndTag: case Token.LF: case Token.EmailAddress: case Token.ImageStartTag: case Token.ImageURL: case Token.ItalicTextStartTag: case Token.ItalicTextEndTag: case Token.ListEndTag: case Token.ListItemEndTag: case Token.ListItemStartTag: case Token.ListStartTag: case Token.OfftopicTextStartTag: case Token.OfftopicTextEndTag: case Token.QuotedTextStartTag: case Token.QuotedTextEndTag: case Token.SizedTextStartTag: case Token.SizedTextEndTag: case Token.SmileyLaugh: case Token.SmileyAngry: case Token.SmileyRegular: case Token.SmileyWink: case Token.SmileyCool: case Token.SmileyTongue: case Token.SmileyConfused: case Token.SmileyShocked: case Token.SmileyDissapointed: case Token.SmileySad: case Token.SmileyEmbarrassed: case Token.StrikedTextStartTag: case Token.StrikedTextEndTag: case Token.Tab: case Token.UnderlinedTextStartTag: case Token.UnderlinedTextEndTag: case Token.URLStartTag: case Token.URI: // Tag or otherwise nonterminal start. Handle statement. First create new nonterminal with current collected text, if available. if(literalTextToken!=null) { NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText); scannedText.Tokens.Add(literalTextToken); literalTextToken=null; _nonTerminalStream.Add(scannedText); index = (_nonTerminalStream.Count - 1); scannedText.CorrespondingEndNTIndex=index; } // token is still in the queue. NonTerminal parsedNonTerminal = ParseNonTerminal(); _nonTerminalStream.Add(parsedNonTerminal); index = (_nonTerminalStream.Count - 1); parsedNonTerminal.CorrespondingEndNTIndex = index; Stack<NonTerminal> ntStack = null; switch(parsedNonTerminal.Type) { case NonTerminalType.BoldTextEnd: ntStack = GetStartNTStack(NonTerminalType.BoldTextStart); break; case NonTerminalType.CodeTextEnd: ntStack = GetStartNTStack(NonTerminalType.CodeTextStart); break; case NonTerminalType.ColoredTextEnd: ntStack = GetStartNTStack(NonTerminalType.ColoredTextStart); break; case NonTerminalType.ItalicTextEnd: ntStack = GetStartNTStack(NonTerminalType.ItalicTextStart); break; case NonTerminalType.ListItemEnd: ntStack = GetStartNTStack(NonTerminalType.ListItemStart); break; case NonTerminalType.ListEnd: ntStack = GetStartNTStack(NonTerminalType.ListStart); break; case NonTerminalType.OfftopicTextEnd: ntStack = GetStartNTStack(NonTerminalType.OfftopicTextStart); break; case NonTerminalType.QuotedTextEnd: ntStack = GetStartNTStack(NonTerminalType.QuotedTextStart); break; case NonTerminalType.SizedTextEnd: ntStack = GetStartNTStack(NonTerminalType.SizedTextStart); break; case NonTerminalType.StrikedTextEnd: ntStack = GetStartNTStack(NonTerminalType.StrikedTextStart); break; case NonTerminalType.UnderlinedTextEnd: ntStack = GetStartNTStack(NonTerminalType.UnderlinedTextStart); break; } if((ntStack!=null) && (ntStack.Count > 0)) { NonTerminal startNT = ntStack.Pop(); startNT.CorrespondingEndNTIndex = index; } break; case Token.CR: // skip. We only use LF tokens for linefeeds. _tokenStream.Dequeue(); break; case Token.ImageEndTag: case Token.URLEndTag: default: // literal text or token which isn't defined properly. Push the complete text as // literal text into the current literal text token. currentToken = _tokenStream.Dequeue(); if(literalTextToken==null) { literalTextToken = new UBBToken(); literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString]; } literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText; break; } } // handle resulting text which is processed but is not transformed to a nonterminal yet. if(literalTextToken!=null) { NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText); scannedText.Tokens.Add(literalTextToken); literalTextToken=null; _nonTerminalStream.Add(scannedText); int index = (_nonTerminalStream.Count - 1); scannedText.CorrespondingEndNTIndex=index; } // change all nonterminals still on one of the stacks into literal text statements, since there are no matching endtokens found. foreach(Stack<NonTerminal> startNTStack in _startNTStacks.Values) { foreach(NonTerminal current in startNTStack) { current.Type = NonTerminalType.LiteralText; } } }