private static bool ProcessHeadingText( ParsingContext context, TokenBuffer tokenBuffer, WikiTextToken.TokenType endingTokenNeeded, StringBuilder headingText) { Contract.Requires(context != null); Contract.Requires(tokenBuffer != null); Contract.Requires(headingText != null); WikiTextToken.TokenType?actualEndingToken = tokenBuffer.ProcessUntilToken( context, t => { switch (t.Type) { case WikiTextToken.TokenType.Text: headingText.Append(t.Text); return(true); case WikiTextToken.TokenType.DoubleApostrophe: case WikiTextToken.TokenType.TripleApostrophe: throw new NotImplementedException("todo next: add support"); default: context.ReportError("Unexpected token in heading definition: {0}".Fmt(t.Text)); return(false); } }, endingTokenNeeded); return(actualEndingToken != null && actualEndingToken == endingTokenNeeded); }
public TokenizerAssert Expect(WikiTextToken.TokenType tokenType, string expectedText) { Assert.AreEqual(tokenType, tokens[cursor].Type); Assert.AreEqual(expectedText, tokens[cursor].Text); cursor++; return(this); }
public WikiTextToken ExpectToken(ParsingContext context, WikiTextToken.TokenType expectedTokenType) { Contract.Requires(context != null); if (EndOfTokens) { context.ReportError("Unexpected end, expected token '{0}'".Fmt(expectedTokenType)); return(null); } WikiTextToken token = Token; if (token.Type != WikiTextToken.TokenType.Text) { context.ReportError("Expected token '{0}' but got '{1}".Fmt(expectedTokenType, token.Type)); return(null); } return(token); }
public WikiTextTokenDef( string tokenString, bool isRegexToken, WikiTextToken.TokenType tokenType, WikiTextTokenScopes availableInScopes, Func <WikiTextTokenScopes, WikiTextTokenScopes> scopeModifierFunc = null) { Contract.Requires(!String.IsNullOrEmpty(tokenString)); if (isRegexToken) { tokenRegex = new Regex(tokenString, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.Compiled); } else { this.tokenString = tokenString; } this.tokenType = tokenType; this.availableInScopes = availableInScopes; this.scopeModifierFunc = scopeModifierFunc; }