private static ImmutableArray <IMarkdownToken> Tokenize(string markdown, string file)
{
var gfm = new GfmEngineBuilder(new Options()).CreateEngine(new HtmlRenderer());
var sourceInfo = SourceInfo.Create(markdown.Replace("\r\n", "\n"), file);
var tokens = gfm.Parser.Tokenize(sourceInfo);
tokens = TokenHelper.CreateParagraghs(
gfm.Parser,
MarkdownParagraphBlockRule.Instance,
tokens,
true,
sourceInfo);
var rewriter =
new MarkdownRewriteEngine(
gfm,
MarkdownTokenRewriterFactory.FromLambda <IMarkdownRewriteEngine, TwoPhaseBlockToken>(
(e, t) => t.Extract(gfm.Parser)));
tokens = rewriter.Rewrite(tokens);
return(tokens);
}
public void TestSourceInfo_Table()
{
const string File = "test.md";
var gfm = new GfmEngineBuilder(new Options()).CreateEngine(new HtmlRenderer());
var tokens = gfm.Parser.Tokenize(
SourceInfo.Create(@"
| H1 | H2 |
|----|----|
|R1C1|R1C2|
|R2C1|R2C2|
".Replace("\r\n", "\n"), File));
var rewriter =
new MarkdownRewriteEngine(
gfm,
MarkdownTokenRewriterFactory.FromLambda <IMarkdownRewriteEngine, TwoPhaseBlockToken>(
(e, t) => t.Extract(gfm.Parser)));
tokens = rewriter.Rewrite(tokens);
Assert.Equal(2, tokens.Length);
Assert.IsType <MarkdownTableBlockToken>(tokens[1]);
var table = (MarkdownTableBlockToken)tokens[1];
Assert.Equal(2, table.Header.Length);
Assert.Equal(2, table.Cells.Length);
Assert.Equal(2, table.SourceInfo.LineNumber);
Assert.Equal(File, table.SourceInfo.File);
Assert.Equal(2, table.Header[0].SourceInfo.LineNumber);
Assert.Equal(2, table.Header[1].SourceInfo.LineNumber);
Assert.Equal(4, table.Cells[0][0].SourceInfo.LineNumber);
Assert.Equal(4, table.Cells[0][1].SourceInfo.LineNumber);
Assert.Equal(5, table.Cells[1][0].SourceInfo.LineNumber);
Assert.Equal(5, table.Cells[1][1].SourceInfo.LineNumber);
}
public void TestSourceInfo_BlockquoteAndList()
{
const string File = "test.md";
var gfm = new GfmEngineBuilder(new Options()).CreateEngine(new HtmlRenderer());
var tokens = gfm.Parser.Tokenize(
SourceInfo.Create(@"> blockquote
> [link text](sometarget)
>
> - list item 1
> - list item 1-1
> - list item 1-2
> - list item 2
>
> more para.
".Replace("\r\n", "\n"), File));
var rewriter =
new MarkdownRewriteEngine(
gfm,
MarkdownTokenRewriterFactory.FromLambda <IMarkdownRewriteEngine, TwoPhaseBlockToken>(
(e, t) => t.Extract(gfm.Parser)));
tokens = rewriter.Rewrite(tokens);
Assert.Equal(1, tokens.Length);
Assert.IsType <MarkdownBlockquoteBlockToken>(tokens[0]);
var blockquote = (MarkdownBlockquoteBlockToken)tokens[0];
Assert.Equal(3, blockquote.Tokens.Length);
Assert.Equal(1, blockquote.SourceInfo.LineNumber);
Assert.Equal(File, blockquote.SourceInfo.File);
Assert.IsType <MarkdownParagraphBlockToken>(blockquote.Tokens[0]);
{
var para = (MarkdownParagraphBlockToken)blockquote.Tokens[0];
Assert.Equal(1, para.SourceInfo.LineNumber);
Assert.Equal(File, para.SourceInfo.File);
Assert.Equal(2, para.InlineTokens.Tokens.Length);
Assert.IsType <MarkdownTextToken>(para.InlineTokens.Tokens[0]);
var text = (MarkdownTextToken)para.InlineTokens.Tokens[0];
Assert.Equal(1, text.SourceInfo.LineNumber);
Assert.Equal(File, text.SourceInfo.File);
Assert.IsType <MarkdownLinkInlineToken>(para.InlineTokens.Tokens[1]);
var link = (MarkdownLinkInlineToken)para.InlineTokens.Tokens[1];
Assert.Equal(2, link.SourceInfo.LineNumber);
Assert.Equal(File, link.SourceInfo.File);
}
Assert.IsType <MarkdownListBlockToken>(blockquote.Tokens[1]);
{
var list = (MarkdownListBlockToken)blockquote.Tokens[1];
Assert.Equal(4, list.SourceInfo.LineNumber);
Assert.Equal(File, list.SourceInfo.File);
Assert.Equal(2, list.Tokens.Length);
Assert.IsType <MarkdownListItemBlockToken>(list.Tokens[0]);
{
var listItem = (MarkdownListItemBlockToken)list.Tokens[0];
Assert.Equal(4, listItem.SourceInfo.LineNumber);
Assert.Equal(File, listItem.SourceInfo.File);
Assert.IsType <MarkdownNonParagraphBlockToken>(listItem.Tokens[0]);
var np = (MarkdownNonParagraphBlockToken)listItem.Tokens[0];
Assert.Equal(1, np.Content.Tokens.Length);
Assert.Equal(4, np.SourceInfo.LineNumber);
Assert.Equal(File, np.SourceInfo.File);
Assert.IsType <MarkdownListBlockToken>(listItem.Tokens[1]);
var subList = (MarkdownListBlockToken)listItem.Tokens[1];
Assert.Equal(2, subList.Tokens.Length);
Assert.IsType <MarkdownListItemBlockToken>(subList.Tokens[0]);
{
var subListItem = (MarkdownListItemBlockToken)subList.Tokens[0];
Assert.Equal(5, subListItem.SourceInfo.LineNumber);
Assert.Equal(File, subListItem.SourceInfo.File);
}
Assert.IsType <MarkdownListItemBlockToken>(subList.Tokens[1]);
{
var subListItem = (MarkdownListItemBlockToken)subList.Tokens[1];
Assert.Equal(6, subListItem.SourceInfo.LineNumber);
Assert.Equal(File, subListItem.SourceInfo.File);
}
}
Assert.IsType <MarkdownListItemBlockToken>(list.Tokens[1]);
{
var listItem = (MarkdownListItemBlockToken)list.Tokens[1];
Assert.Equal(7, listItem.SourceInfo.LineNumber);
Assert.Equal(File, listItem.SourceInfo.File);
}
}
Assert.IsType <MarkdownParagraphBlockToken>(blockquote.Tokens[2]);
{
var para = (MarkdownParagraphBlockToken)blockquote.Tokens[2];
Assert.Equal(9, para.SourceInfo.LineNumber);
Assert.Equal(File, para.SourceInfo.File);
}
}
public void TestSourceInfo_Basic()
{
const string File = "test.md";
var gfm = new GfmEngineBuilder(new Options()).CreateEngine(new HtmlRenderer());
var tokens = gfm.Parser.Tokenize(
SourceInfo.Create(@"
# HEAD1
First line.
More line.
## HEAD2
Yeah!".Replace("\r\n", "\n"), File));
var rewriter =
new MarkdownRewriteEngine(
gfm,
MarkdownTokenRewriterFactory.FromLambda <IMarkdownRewriteEngine, TwoPhaseBlockToken>(
(e, t) => t.Extract(gfm.Parser)));
tokens = rewriter.Rewrite(tokens);
Assert.Equal(5, tokens.Length);
Assert.IsType <MarkdownNewLineBlockToken>(tokens[0]);
Assert.IsType <MarkdownHeadingBlockToken>(tokens[1]);
Assert.IsType <MarkdownParagraphBlockToken>(tokens[2]);
Assert.IsType <MarkdownHeadingBlockToken>(tokens[3]);
Assert.IsType <MarkdownParagraphBlockToken>(tokens[4]);
var para = (MarkdownParagraphBlockToken)tokens[2];
Assert.Equal(3, para.InlineTokens.Tokens.Length);
Assert.IsType <MarkdownTextToken>(para.InlineTokens.Tokens[0]);
Assert.IsType <MarkdownBrInlineToken>(para.InlineTokens.Tokens[1]);
Assert.IsType <MarkdownTextToken>(para.InlineTokens.Tokens[2]);
Assert.Equal(1, tokens[0].SourceInfo.LineNumber);
Assert.Equal(File, tokens[0].SourceInfo.File);
Assert.Equal("\n\n", tokens[0].SourceInfo.Markdown);
Assert.Equal(3, tokens[1].SourceInfo.LineNumber);
Assert.Equal(File, tokens[1].SourceInfo.File);
Assert.Equal("# HEAD1\n", tokens[1].SourceInfo.Markdown);
Assert.Equal(4, tokens[2].SourceInfo.LineNumber);
Assert.Equal(File, tokens[2].SourceInfo.File);
Assert.Equal("First line. \nMore line.\n", tokens[2].SourceInfo.Markdown);
Assert.Equal(4, para.InlineTokens.Tokens[0].SourceInfo.LineNumber);
Assert.Equal(File, para.InlineTokens.Tokens[0].SourceInfo.File);
Assert.Equal("First line.", para.InlineTokens.Tokens[0].SourceInfo.Markdown);
Assert.Equal(4, para.InlineTokens.Tokens[1].SourceInfo.LineNumber);
Assert.Equal(File, para.InlineTokens.Tokens[1].SourceInfo.File);
Assert.Equal(" \n", para.InlineTokens.Tokens[1].SourceInfo.Markdown);
Assert.Equal(5, para.InlineTokens.Tokens[2].SourceInfo.LineNumber);
Assert.Equal(File, para.InlineTokens.Tokens[2].SourceInfo.File);
Assert.Equal("More line.", para.InlineTokens.Tokens[2].SourceInfo.Markdown);
Assert.Equal(6, tokens[3].SourceInfo.LineNumber);
Assert.Equal(File, tokens[3].SourceInfo.File);
Assert.Equal("## HEAD2\n", tokens[3].SourceInfo.Markdown);
Assert.Equal(7, tokens[4].SourceInfo.LineNumber);
Assert.Equal(File, tokens[4].SourceInfo.File);
Assert.Equal("Yeah!", tokens[4].SourceInfo.Markdown);
}