private static bool HandleExternalLinkUrlTokenInText( ParsingContext context, WikiTextToken token, ref TextParsingMode parsingMode, ref Uri externalLinkUrl) { if (parsingMode != TextParsingMode.ExternalLinkUrl) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } string uriString = token.Text.Trim(); try { externalLinkUrl = new Uri(uriString); } catch (UriFormatException ex) { context.ReportError("Invalid external link URL '{0}': {1}".Fmt(uriString, ex.Message)); return(false); } parsingMode = TextParsingMode.ExternalLinkDescription; return(true); }
private static bool ProcessHeadingAnchor(ParsingContext context, TokenBuffer tokenBuffer, out string anchorId) { Contract.Requires(context != null); Contract.Requires(tokenBuffer != null); anchorId = null; if (!tokenBuffer.EndOfTokens) { WikiTextToken token = tokenBuffer.Token; if (token.Type == WikiTextToken.TokenType.HeadingAnchor) { tokenBuffer.MoveToNextToken(); if (!FetchHeadingAnchor(context, tokenBuffer, out anchorId)) { return(false); } } else { context.ReportError("Unexpected tokens after ending heading token"); return(false); } } return(true); }
private static bool FetchHeadingAnchor(ParsingContext context, TokenBuffer tokenBuffer, out string anchorId) { Contract.Requires(context != null); Contract.Requires(tokenBuffer != null); anchorId = null; WikiTextToken token = tokenBuffer.ExpectToken(context, WikiTextToken.TokenType.Text); if (token == null) { return(false); } string potentialAnchorId = token.Text; if (!ValidateAnchor(potentialAnchorId)) { context.ReportError("Invalid heading anchor ID: '{0}'".Fmt(potentialAnchorId)); return(false); } anchorId = token.Text; tokenBuffer.MoveToNextToken(); return(true); }
public WikiTextToken.TokenType?ProcessUntilToken( ParsingContext context, Func <WikiTextToken, bool> tokenFunc, params WikiTextToken.TokenType[] untilTypes) { Contract.Requires(context != null); Contract.Requires(tokenFunc != null); while (!EndOfTokens) { WikiTextToken token = Token; if (untilTypes.Contains(token.Type)) { return(token.Type); } if (!tokenFunc(token)) { return(null); } MoveToNextToken(); } context.ReportError("Expected one of token types ({0}), but they are missing".Fmt(untilTypes.Concat(x => x.ToString(), ","))); return(null); }
private static bool HandleDoubleSquareBracketsCloseTokenInText( DocumentDefBuilder docBuilder, ParsingContext context, ref TextParsingMode parsingMode, WikiTextToken token, InternalLinkIdBuilder linkIdBuilder, StringBuilder internalLinkDescription) { Contract.Requires(docBuilder != null); Contract.Requires(context != null); Contract.Requires(token != null); Contract.Requires(linkIdBuilder != null); if (parsingMode != TextParsingMode.InternalLinkPageName && parsingMode != TextParsingMode.InternalLinkDescription) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } parsingMode = TextParsingMode.RegularText; bool result = AddInternalLink(docBuilder, context, linkIdBuilder, internalLinkDescription); linkIdBuilder.Clear(); if (internalLinkDescription != null) { internalLinkDescription.Clear(); } return(result); }
private static void AddTextTokenIfAny( string wikiText, ICollection <WikiTextToken> tokens, ref int?textTokenStart, int index, WikiTextTokenScopes scope) { Contract.Requires(wikiText != null); Contract.Requires(tokens != null); Contract.Requires(textTokenStart == null || (textTokenStart != null && index > textTokenStart.Value && textTokenStart.Value < wikiText.Length && index <= wikiText.Length)); Contract.Ensures(textTokenStart == null); if (!textTokenStart.HasValue) { return; } WikiTextToken prevToken = new WikiTextToken( WikiTextToken.TokenType.Text, wikiText.Substring(textTokenStart.Value, index - textTokenStart.Value), scope); tokens.Add(prevToken); textTokenStart = null; }
private static bool HandleExternalLinkUrlLeadingSpaceTokenInText( ParsingContext context, WikiTextToken token, TextParsingMode parsingMode) { if (parsingMode != TextParsingMode.ExternalLinkUrl) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } return(true); }
private static bool HandlePipeTokenInText(ParsingContext context, WikiTextToken token, ref TextParsingMode parsingMode) { Contract.Requires(context != null); Contract.Requires(token != null); if (parsingMode != TextParsingMode.InternalLinkPageName) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } parsingMode = TextParsingMode.InternalLinkDescription; return(true); }
private void ProcessLine( DocumentDefBuilder docBuilder, ParsingContext context, string lineText) { Contract.Requires(docBuilder != null); Contract.Requires(lineText != null); WikiTokenizationSettings tokenizationSettings = new WikiTokenizationSettings(); tokenizationSettings.IsWholeLine = true; IList <WikiTextToken> tokens = tokenizer.TokenizeWikiText(lineText, tokenizationSettings); if (tokens.Count == 0) { throw new NotImplementedException("todo next:"); } TokenBuffer tokenBuffer = new TokenBuffer(tokens); WikiTextToken firstToken = tokenBuffer.Token; switch (firstToken.Type) { case WikiTextToken.TokenType.Heading1Start: case WikiTextToken.TokenType.Heading2Start: case WikiTextToken.TokenType.Heading3Start: case WikiTextToken.TokenType.Heading4Start: case WikiTextToken.TokenType.Heading5Start: case WikiTextToken.TokenType.Heading6Start: docBuilder.FinalizeCurrentParagraph(); HandleHeadingLine(docBuilder, context, tokenBuffer); break; case WikiTextToken.TokenType.Text: case WikiTextToken.TokenType.DoubleApostrophe: case WikiTextToken.TokenType.TripleApostrophe: case WikiTextToken.TokenType.DoubleSquareBracketsOpen: case WikiTextToken.TokenType.SingleSquareBracketsOpen: case WikiTextToken.TokenType.BulletList: case WikiTextToken.TokenType.NumberedList: case WikiTextToken.TokenType.Indent: HandleText(docBuilder, context, tokenBuffer); break; default: throw new NotImplementedException("todo next: {0}".Fmt(firstToken.Type)); } }
private static bool HandleSingleSquareBracketsOpenTokenInText( ParsingContext context, WikiTextToken token, ref TextParsingMode parsingMode) { Contract.Requires(context != null); Contract.Requires(token != null); if (parsingMode != TextParsingMode.RegularText) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } parsingMode = TextParsingMode.ExternalLinkUrl; return(true); }
public bool ProcessUntilEnd(Func <WikiTextToken, bool> tokenFunc) { Contract.Requires(tokenFunc != null); while (!EndOfTokens) { WikiTextToken token = Token; if (!tokenFunc(token)) { return(false); } MoveToNextToken(); } return(true); }
private static bool HandleSingleSquareBracketsCloseTokenInText( DocumentDefBuilder docBuilder, ParsingContext context, ref TextParsingMode parsingMode, WikiTextToken token, Uri externalLinkUrl, StringBuilder textBuilder) { if (parsingMode != TextParsingMode.ExternalLinkUrl && parsingMode != TextParsingMode.ExternalLinkDescription) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } parsingMode = TextParsingMode.RegularText; return(AddExternalLink(docBuilder, externalLinkUrl, textBuilder)); }
private static bool HandleTextTokenInText( DocumentDefBuilder docBuilder, TextParsingMode parsingMode, WikiTextToken token, bool isFirstElementOfContinuedLine, InternalLinkIdBuilder linkIdBuilder, ref StringBuilder textBuilder) { Contract.Requires(docBuilder != null); Contract.Requires(token != null); Contract.Requires(linkIdBuilder != null); switch (parsingMode) { case TextParsingMode.RegularText: docBuilder.AddTextToParagraph(token.Text, isFirstElementOfContinuedLine); return(true); case TextParsingMode.InternalLinkPageName: linkIdBuilder.AppendText(token.Text); return(true); case TextParsingMode.InternalLinkDescription: if (textBuilder == null) { textBuilder = new StringBuilder(); } textBuilder.Append(token.Text); return(true); case TextParsingMode.ExternalLinkDescription: if (textBuilder == null) { textBuilder = new StringBuilder(); } textBuilder.Append(token.Text); return(true); default: throw new NotImplementedException("todo next:"); } }
public WikiTextToken ExpectToken(ParsingContext context, WikiTextToken.TokenType expectedTokenType) { Contract.Requires(context != null); if (EndOfTokens) { context.ReportError("Unexpected end, expected token '{0}'".Fmt(expectedTokenType)); return(null); } WikiTextToken token = Token; if (token.Type != WikiTextToken.TokenType.Text) { context.ReportError("Expected token '{0}' but got '{1}".Fmt(expectedTokenType, token.Type)); return(null); } return(token); }
private static bool HandleIndentTokenInText(DocumentDefBuilder docBuilder, WikiTextToken token) { Contract.Requires(docBuilder != null); docBuilder.StartNewParagraph(ParagraphElement.ParagraphType.Regular, token.Text.Length); return(true); }
private static bool HandleNamespaceSeparatorTokenInText( ParsingContext context, TextParsingMode parsingMode, InternalLinkIdBuilder linkIdBuilder, WikiTextToken token) { Contract.Requires(context != null); Contract.Requires(linkIdBuilder != null); Contract.Requires(token != null); if (parsingMode != TextParsingMode.InternalLinkPageName) { context.ReportError("Token {0} is not allowed here".Fmt(token.Text)); return(false); } linkIdBuilder.AddSeparator(); return(true); }
private static void HandleHeadingLine(DocumentDefBuilder docBuilder, ParsingContext context, TokenBuffer tokenBuffer) { Contract.Requires(docBuilder != null); Contract.Requires(context != null); Contract.Requires(tokenBuffer != null); WikiTextToken firstToken = tokenBuffer.Token; WikiTextToken.TokenType endingTokenNeeded; int headingLevel; switch (firstToken.Type) { case WikiTextToken.TokenType.Heading1Start: headingLevel = 1; endingTokenNeeded = WikiTextToken.TokenType.Heading1End; break; case WikiTextToken.TokenType.Heading2Start: headingLevel = 2; endingTokenNeeded = WikiTextToken.TokenType.Heading2End; break; case WikiTextToken.TokenType.Heading3Start: headingLevel = 3; endingTokenNeeded = WikiTextToken.TokenType.Heading3End; break; case WikiTextToken.TokenType.Heading4Start: headingLevel = 4; endingTokenNeeded = WikiTextToken.TokenType.Heading4End; break; case WikiTextToken.TokenType.Heading5Start: headingLevel = 5; endingTokenNeeded = WikiTextToken.TokenType.Heading5End; break; case WikiTextToken.TokenType.Heading6Start: headingLevel = 6; endingTokenNeeded = WikiTextToken.TokenType.Heading6End; break; default: throw new InvalidOperationException("BUG"); } tokenBuffer.MoveToNextToken(); StringBuilder headingText = new StringBuilder(); if (!ProcessHeadingText(context, tokenBuffer, endingTokenNeeded, headingText)) { return; } HeadingElement headingEl = new HeadingElement(headingText.ToString().Trim(), headingLevel); docBuilder.AddRootChild(headingEl); tokenBuffer.MoveToNextToken(); string anchorId; if (!ProcessHeadingAnchor(context, tokenBuffer, out anchorId)) { return; } tokenBuffer.ProcessUntilEnd( t => { context.ReportError("Unexpected token at the end of heading"); return(false); }); headingEl.AnchorId = anchorId; }