/// <summary>
/// Creates a new token object using this definition and prefills it.
/// </summary>
/// <returns>An TDLToken token</returns>
public IToken CreateTokenFromDefinition()
{
UBBToken toReturn = new UBBToken();
toReturn.RelatedTokenDefinition = this;
return(toReturn);
}
/// <summary>
/// Creates a new token object using this definition and prefills it.
/// </summary>
/// <returns>An TDLToken token</returns>
public IToken CreateTokenFromDefinition()
{
UBBToken toReturn = new UBBToken();
toReturn.RelatedTokenDefinition = this;
return toReturn;
}
/// <summary>
/// Converts the passed in set of popped tokens to one nonterminal of type LiteralText. This
/// routine is used when an error is found in the stream so the currently popped tokens are
/// apparantly not part of a statement, therefor should be converted to a literal string nonterminal,
/// which will be transformed by the interpreter as a textstream. The error causing token is <i>not</i> included
/// in the tokensPopped collection.
/// </summary>
/// <param name="tokensPopped">Set of tokens popped, EXCLUDING the error causing token, which will be used as one set
/// of tokens of one nonterminal of type literal text</param>
/// <returns>the literal text nonterminal with all the tokens passed in.</returns>
private NonTerminal ConvertPoppedTokensToLiteralText(List<IToken> tokensPopped)
{
NonTerminal toReturn = new NonTerminal(NonTerminalType.LiteralText);
IToken literalTextToken = new UBBToken();
literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString];
IToken currentToken = null;
for(int i=0;i<tokensPopped.Count;i++)
{
currentToken = (IToken)tokensPopped[i];
literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText;
}
toReturn.Tokens.Add(literalTextToken);
return toReturn;
}
/// <summary>
/// Parses the token stream into nonterminal objects. Uses simple LL(1) algo.
/// Fills _nonTerminalStream;
/// </summary>
private void ParseTokens()
{
IToken literalTextToken = null;
// parse the tokens
bool eofFound = false;
while((_tokenStream.Count > 0)&&!eofFound)
{
IToken currentToken = (IToken)_tokenStream.Peek();
int index = 0;
switch((Token)currentToken.TokenID)
{
case Token.EOF:
// done
eofFound = true;
break;
case Token.BoldTextStartTag:
case Token.BoldTextEndTag:
case Token.CodeTextStartTag:
case Token.CodeTextEndTag:
case Token.ColoredTextStartTag:
case Token.ColoredTextEndTag:
case Token.LF:
case Token.EmailAddress:
case Token.ImageStartTag:
case Token.ImageURL:
case Token.ItalicTextStartTag:
case Token.ItalicTextEndTag:
case Token.ListEndTag:
case Token.ListItemEndTag:
case Token.ListItemStartTag:
case Token.ListStartTag:
case Token.OfftopicTextStartTag:
case Token.OfftopicTextEndTag:
case Token.QuotedTextStartTag:
case Token.QuotedTextEndTag:
case Token.SizedTextStartTag:
case Token.SizedTextEndTag:
case Token.SmileyLaugh:
case Token.SmileyAngry:
case Token.SmileyRegular:
case Token.SmileyWink:
case Token.SmileyCool:
case Token.SmileyTongue:
case Token.SmileyConfused:
case Token.SmileyShocked:
case Token.SmileyDissapointed:
case Token.SmileySad:
case Token.SmileyEmbarrassed:
case Token.StrikedTextStartTag:
case Token.StrikedTextEndTag:
case Token.Tab:
case Token.UnderlinedTextStartTag:
case Token.UnderlinedTextEndTag:
case Token.URLStartTag:
case Token.URI:
// Tag or otherwise nonterminal start. Handle statement. First create new nonterminal with current collected text, if available.
if(literalTextToken!=null)
{
NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText);
scannedText.Tokens.Add(literalTextToken);
literalTextToken=null;
_nonTerminalStream.Add(scannedText);
index = (_nonTerminalStream.Count - 1);
scannedText.CorrespondingEndNTIndex=index;
}
// token is still in the queue.
NonTerminal parsedNonTerminal = ParseNonTerminal();
_nonTerminalStream.Add(parsedNonTerminal);
index = (_nonTerminalStream.Count - 1);
parsedNonTerminal.CorrespondingEndNTIndex = index;
Stack<NonTerminal> ntStack = null;
switch(parsedNonTerminal.Type)
{
case NonTerminalType.BoldTextEnd:
ntStack = GetStartNTStack(NonTerminalType.BoldTextStart);
break;
case NonTerminalType.CodeTextEnd:
ntStack = GetStartNTStack(NonTerminalType.CodeTextStart);
break;
case NonTerminalType.ColoredTextEnd:
ntStack = GetStartNTStack(NonTerminalType.ColoredTextStart);
break;
case NonTerminalType.ItalicTextEnd:
ntStack = GetStartNTStack(NonTerminalType.ItalicTextStart);
break;
case NonTerminalType.ListItemEnd:
ntStack = GetStartNTStack(NonTerminalType.ListItemStart);
break;
case NonTerminalType.ListEnd:
ntStack = GetStartNTStack(NonTerminalType.ListStart);
break;
case NonTerminalType.OfftopicTextEnd:
ntStack = GetStartNTStack(NonTerminalType.OfftopicTextStart);
break;
case NonTerminalType.QuotedTextEnd:
ntStack = GetStartNTStack(NonTerminalType.QuotedTextStart);
break;
case NonTerminalType.SizedTextEnd:
ntStack = GetStartNTStack(NonTerminalType.SizedTextStart);
break;
case NonTerminalType.StrikedTextEnd:
ntStack = GetStartNTStack(NonTerminalType.StrikedTextStart);
break;
case NonTerminalType.UnderlinedTextEnd:
ntStack = GetStartNTStack(NonTerminalType.UnderlinedTextStart);
break;
}
if((ntStack!=null) && (ntStack.Count > 0))
{
NonTerminal startNT = ntStack.Pop();
startNT.CorrespondingEndNTIndex = index;
}
break;
case Token.CR:
// skip. We only use LF tokens for linefeeds.
_tokenStream.Dequeue();
break;
case Token.ImageEndTag:
case Token.URLEndTag:
default:
// literal text or token which isn't defined properly. Push the complete text as
// literal text into the current literal text token.
currentToken = _tokenStream.Dequeue();
if(literalTextToken==null)
{
literalTextToken = new UBBToken();
literalTextToken.RelatedTokenDefinition = Parser.TokenDefinitions[(int)Token.UntokenizedLiteralString];
}
literalTextToken.LiteralMatchedTokenText += currentToken.LiteralMatchedTokenText;
break;
}
}
// handle resulting text which is processed but is not transformed to a nonterminal yet.
if(literalTextToken!=null)
{
NonTerminal scannedText = new NonTerminal(NonTerminalType.LiteralText);
scannedText.Tokens.Add(literalTextToken);
literalTextToken=null;
_nonTerminalStream.Add(scannedText);
int index = (_nonTerminalStream.Count - 1);
scannedText.CorrespondingEndNTIndex=index;
}
// change all nonterminals still on one of the stacks into literal text statements, since there are no matching endtokens found.
foreach(Stack<NonTerminal> startNTStack in _startNTStacks.Values)
{
foreach(NonTerminal current in startNTStack)
{
current.Type = NonTerminalType.LiteralText;
}
}
}
