public async Task BuildSite(Site site)
{
foreach (var version in site.Versions)
{
// compose doc sections
foreach (var section in site.GetSectionsByVersion(version))
{
var uiBundleRef = LinkParser.Parse("xref://ui-bundle:tanka-docs-section.yml").Xref !.Value;
var uiContent = site.GetSectionByXref(uiBundleRef, section);
if (uiContent == null)
{
throw new InvalidOperationException($"Could not resolve ui-bundle. Xref '{uiBundleRef}' could not be resolved.'");
}
var uiBundle = new HandlebarsUiBundle(site, uiContent, _output);
await uiBundle.Initialize(CancellationToken.None);
var composer = new SectionComposer(site, _cache, _output, uiBundle);
await composer.ComposeSection(section);
}
}
await ComposeIndexPage(site);
}
public void Should_replace_link_tags()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
parser.ParsedText.ShouldEqual("some text some more text and yet some more");
links.Length.ShouldEqual(2);
}
static void Main(string[] args)
{
var parser = new LinkParser();
var domains = new string[]
{
"https://mail.ru/"
};
var strategies = new ILinkParserStrategy[]
{
new BaseParserStrategy(),
new ExcludeValueParserStrategie("profile"),
new ImageSrcParserStrategy(),
new NotLessThenPageSizeParserStrategy(1024 * 200)
};
Console.CancelKeyPress += new ConsoleCancelEventHandler((object sender, ConsoleCancelEventArgs cancelArgs) =>
{
parser.Stop();
cancelArgs.Cancel = true;
});
parser.Parse(domains, "result.txt", strategies, new ProgressReporter());
Console.WriteLine("Parsing is over");
Console.ReadLine();
}
private void setInfoText(string text)
{
try
{
int previousHeight = linkLabelInfo.Height;
var parser = new LinkParser(text);
var links = parser.Parse();
if (parser.ParsedText.Length > GDI_SIZE_LIMIT)
{
linkLabelInfo.Text = parser.ParsedText.Substring(0, GDI_SIZE_LIMIT);
}
else
{
linkLabelInfo.Text = parser.ParsedText;
}
linkLabelInfo.LinkArea = new LinkArea(0, 0);
foreach (var link in links)
{
if (link.Start + link.Length <= GDI_SIZE_LIMIT)
{
linkLabelInfo.Links.Add(link.Start, link.Length);
}
}
var difference = linkLabelInfo.Height - previousHeight;
Height = Height + difference;
}
catch (Exception exception)
{
_bus.Publish(new ErrorMessage(exception));
}
}
public void Should_replace_link_tags()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
parser.ParsedText.ShouldEqual("some text some more text and yet some more");
links.Length.ShouldEqual(2);
}
public void Should_return_links()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
links.Length.ShouldEqual(2);
parser.ParsedText.Substring(links[0].Start, links[0].Length).ShouldEqual("some more text");
parser.ParsedText.Substring(links[1].Start, links[1].Length).ShouldEqual("some more");
}
public void Should_return_links()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
links.Length.ShouldEqual(2);
parser.ParsedText.Substring(links[0].Start, links[0].Length).ShouldEqual("some more text");
parser.ParsedText.Substring(links[1].Start, links[1].Length).ShouldEqual("some more");
}
private void should_grab_links()
{
var input = File.ReadAllText(_testFilePath);
var linkGrabber = new LinkParser();
var result = linkGrabber.Parse(input);
Approvals.VerifyAll(result, "[name, link]");
}
public async Task <IActionResult> Index(int page)
{
var houses = await httpWrapper.GetAsync <List <House> >(RelativePaths.HousesEndpoint, page);
var navigationControls = LinkParser.Parse(houses.LinkHeader);
ViewBag.controls = new NavigationControls(navigationControls);
return(View(houses.HttpResponse));
}
public void Should_return_links()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
links.Length.ShouldEqual(2);
// some more text
links[0].Start.ShouldEqual(10);
links[0].Length.ShouldEqual(14);
// some more
links[1].Start.ShouldEqual(33);
links[1].Length.ShouldEqual(9);
}
public void ParseUri()
{
/* Given */
var link = ExternalLink;
/* When */
var definition = LinkParser.Parse(link);
/* Then */
Assert.True(definition.IsExternal);
Assert.False(definition.IsXref);
Assert.Equal("https://link.invalid", definition.Uri);
}
public void ParseXref_with_SectionId()
{
/* Given */
var link = XrefWithSectionIdLink;
/* When */
var definition = LinkParser.Parse(link);
/* Then */
Assert.True(definition.IsXref);
Assert.False(definition.IsExternal);
Assert.Equal("link.md", definition.Xref?.Path);
Assert.Equal("section", definition.Xref?.SectionId);
}
public object?ReadYaml(IParser parser, Type type)
{
// should be string
var value = parser.Consume <Scalar>().Value;
if (string.IsNullOrEmpty(value))
{
return(null);
}
var link = LinkParser.Parse(value);
return(link);
}
public void SetText(string text, string caption)
{
Text = caption;
var parser = new LinkParser(text);
var links = parser.Parse();
linkLabel.Text = parser.ParsedText;
linkLabel.LinkArea = new LinkArea(0, 0);
foreach (var link in links)
{
linkLabel.Links.Add(link.Start, link.Length);
}
Height = linkLabel.Height + (linkLabel.Top * 2) + 28;
Width = linkLabel.Width + (linkLabel.Left * 2) + 8;
}
public void ParseXref_with_SectionIdAndVersionAndQuery1()
{
/* Given */
var link = XrefWithSectionIdAndVersionAndQuery;
/* When */
var definition = LinkParser.Parse(link);
/* Then */
Assert.True(definition.IsXref);
Assert.False(definition.IsExternal);
Assert.Equal("link.md", definition.Xref?.Path);
Assert.Equal("section", definition.Xref?.SectionId);
Assert.Equal("release/1.0.0", definition.Xref?.Version);
Assert.Equal(3, definition.Xref?.Query.Count);
}
public void ParseXref_with_SectionIdAndVersionAndQuery2()
{
/* Given */
var link = "xref://section@release/1.0.0:link.md?a=1";
/* When */
var definition = LinkParser.Parse(link);
/* Then */
Assert.True(definition.IsXref);
Assert.False(definition.IsExternal);
Assert.Equal("link.md", definition.Xref?.Path);
Assert.Equal("section", definition.Xref?.SectionId);
Assert.Equal("release/1.0.0", definition.Xref?.Version);
Assert.Single(definition.Xref?.Query, kv => kv.Key == "a" && kv.Value == "1");
Assert.Equal(1, definition.Xref?.Query.Count);
}
public IEnumerable <PageCrawl> Start()
{
while (frontier.Count > 0)
{
Console.WriteLine("Frontier size: " + frontier.Count);
Page currentCrawlPage = frontier.Dequeue();
if (ShouldSkip(currentCrawlPage.Uri))
{
continue;
}
latestCrawl = Browser.Crawl(currentCrawlPage);
currentCrawlPage.Crawl = latestCrawl;
visited.Add(currentCrawlPage.ToString());
// Ensure crawl didn't fail
if (latestCrawl == null)
{
continue;
}
Uri[] newLinks = LinkParser.Parse(latestCrawl.Content, latestCrawl.Page.Uri);
// Add links and new crawl
foreach (Uri uri in newLinks)
{
if (!visited.Contains(uri.ToString()))
{
frontier.Enqueue(new Page(uri, currentCrawlPage));
}
}
yield return(latestCrawl);
int millisecondDelay = robots[currentCrawlPage.Uri.Authority].CrawlDelay * 1000;
millisecondDelay = (int)Math.Max(0, millisecondDelay - (DateTime.Now - latestCrawl.CrawlTime).TotalMilliseconds);
Console.WriteLine("Waiting for " + millisecondDelay + "ms");
Thread.Sleep(millisecondDelay);
}
}
private List <Link> getLinks()
{
Text = Text.Trim();
if (Text.EndsWith(Environment.NewLine))
{
Text = Text.Substring(0, Text.Length - Environment.NewLine.Length);
}
var parser = new LinkParser(Text);
var links = parser.Parse();
Text = parser.ParsedText;
var textForLambda = Text;
return(links
.Where(x => containsLink(textForLambda.Substring(x.Start, x.Length)))
.Select(x =>
new Link(
getBeginningOfLine(textForLambda, x.Start),
getEndOfLine(textForLambda, x.Start + x.Length),
getFile(textForLambda.Substring(x.Start, x.Length)),
getLine(textForLambda.Substring(x.Start, x.Length)))).ToList());
}
public HandlebarsPageRenderer(string templateHbs, IReadOnlyDictionary <string, string> partials, DocsSiteRouter router)
{
_templateHbs = templateHbs;
_handlebars = Handlebars.Create();
_handlebars.RegisterHelper("link_to", (output, options, context, arguments) =>
{
if (arguments.Length != 1)
{
throw new InvalidOperationException($"link_to requires one argument of type DisplayLink");
}
var target = arguments[0];
string?url = null;
string?title = null;
if (target is DisplayLink displayLink)
{
title = displayLink.Title ?? "";
url = displayLink.Link.Xref != null
? router.GenerateRoute(displayLink.Link.Xref.Value)
: displayLink.Link.Uri;
}
if (url == null)
{
url = "[TODO: MISSING LINK TARGET]";
}
options.Template(output, new
{
title,
url
});
});
_handlebars.RegisterHelper("xref", (output, options, context, arguments) =>
{
if (arguments.Length != 1)
{
throw new InvalidOperationException($"xref requires one argument of type xref");
}
var target = arguments[0];
string?url = null;
if (target is string xrefStr)
{
target = LinkParser.Parse(xrefStr);
}
if (target is Link link)
{
if (link.IsExternal)
{
url = link.Uri;
}
else
{
target = link.Xref !.Value;
}
private void setInfoText(string text)
{
try
{
int previousHeight = linkLabelInfo.Height;
var parser = new LinkParser(text);
var links = parser.Parse();
if (parser.ParsedText.Length > GDI_SIZE_LIMIT)
linkLabelInfo.Text = parser.ParsedText.Substring(0, GDI_SIZE_LIMIT);
else
linkLabelInfo.Text = parser.ParsedText;
linkLabelInfo.LinkArea = new LinkArea(0, 0);
foreach (var link in links)
{
if (link.Start + link.Length <= GDI_SIZE_LIMIT)
linkLabelInfo.Links.Add(link.Start, link.Length);
}
var difference = linkLabelInfo.Height - previousHeight;
Height = Height + difference;
}
catch (Exception exception)
{
_bus.Publish(new ErrorMessage(exception));
}
}
public void LinkWithMultipleParameters()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse("</raw/hb>; rel=\"up\"; riaktag=\"foo\"");
Assert.AreEqual(1, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("foo", links[0].RiakTag);
// we don't strip rel="up" out here since there is a riaktag too.
Assert.AreEqual("</raw/hb>; rel=\"up\"; riaktag=\"foo\"",
links.ToString());
}
public void LinkWithUnknownParameters()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse("</raw/hb>; rel=\"up\"; riaktag=\"foo\"; unk=\"value\"");
Assert.AreEqual(1, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("foo", links[0].RiakTag);
Assert.AreEqual("value", links[0].UnknownParameters["unk"]);
// normalizes as unknown -> rel -> riaktag
Assert.AreEqual("</raw/hb>; unk=\"value\"; rel=\"up\"; riaktag=\"foo\"",
links.ToString());
}
public void LinkWithRelationalUri()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse("</raw/hb>; rel=\"up\", </raw/hb/fourth>; riaktag=\"foo\"");
Assert.AreEqual(2, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("</raw/hb>; rel=\"up\"", links[0].ToString());
Assert.AreEqual("/raw/hb/fourth", links[1].UriResource);
Assert.AreEqual("foo", links[1].RiakTag);
Assert.AreEqual("</raw/hb/fourth>; riaktag=\"foo\"", links[1].ToString());
// we strip out the rel="up" link if it is the only thing in the link
Assert.AreEqual("</raw/hb/fourth>; riaktag=\"foo\"",
links.ToString());
}
public void LinkOddlyFormedIsParsedAndNormalized()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse(" </raw/hb> ;rel=\"up\" ; riaktag=\"foo\" ;");
Assert.AreEqual(1, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("foo", links[0].RiakTag);
Assert.AreEqual("</raw/hb>; rel=\"up\"; riaktag=\"foo\"",
links.ToString());
}