private void SetTovarCode(IntercarsProfile profile)
{
ReqParametres req = new ReqParametres($"https://ic-ua.intercars.eu/dynamic/uni/ws_towary.php?wit=ICKATALOGWEB&pro=&kraj=UA&oesearch={profile.Number}&ofe=", HttpMethod.POST, $"oesearch={profile.Number}");
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.34 Safari/537.36");
SetCookies(req);
req.RowRequest.ContentType = "application/x-www-form-urlencoded";
LinkParser link = new LinkParser(req.Request);
List <string> allCodes = link.Data.Replace("\n", "").Replace("\r", "").ParsRegex("Daj_Katalog_Detail_Clob(.*?);", 1);
List <string> uniqCodes = new List <string>();
allCodes.ForEach(
code =>
{
string rawCode = code.Split(',')[code.Split(',').Length - 1].ParsFromTo("'", "'");
if (!uniqCodes.Contains(rawCode) && Regex.IsMatch(rawCode, "[A-Z0-9]+"))
{
uniqCodes.Add(rawCode);
}
});
if (uniqCodes.Count > 0)
{
profile.TovarCode = uniqCodes[0];
}
else
{
profile.TovarCode = "";
}
}
private void SetParametres(IntercarsProfile profile)
{
ReqParametres req = new ReqParametres("https://ic-ua.intercars.eu/dynamic/uni/ws_towDetail.php?wit=ICKATALOGWEB&p=F", HttpMethod.POST, $"artnr={profile.Number}&witryna=ICKATALOGWEB&towkod={profile.TovarCode}&lang=RU&nb=N&kraj=UA&typ=&wsk=");
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.34 Safari/537.36");
SetCookies(req);
req.RowRequest.ContentType = "application/x-www-form-urlencoded";
LinkParser link = new LinkParser(req.Request);
profile.Description = link.Data.ParsFromTo("<title>", "<");
profile.Image = link.Data.ParsFromTo("<img src=\"", "\"");
if (profile.Image != "")
{
profile.Image = "https://ic-ua.intercars.eu" + profile.Image;
}
SetModel(profile, link.Data);
SetZaminniki(profile, link.Data);
SetOriginalNumbers(profile);
List <string> additionalInformation = link.Data.ParsRegex("dKartaNazClass(.*?)>(.*?)<", 2);
for (int i = 0; i < additionalInformation.Count; i += 2)
{
if (i == 0)
{
profile.AdditionalInformation = additionalInformation[i] + " " + additionalInformation[i + 1];
}
else
{
profile.AdditionalInformation += "\n" + additionalInformation[i] + " " + additionalInformation[i + 1];
}
}
SetPrices(profile, link.Data);
}
private void SetOriginalNumbers(IntercarsProfile profile)
{
if (profile.NomerOe == "")
{
return;
}
ReqParametres req = new ReqParametres($"https://ic-ua.intercars.eu/dynamic/ickatalogweb/ws_getsoap.php?call=numoe&art={profile.NomerOe}&wit=ICKATALOGWEB", HttpMethod.POST, $"call=numoe&art={profile.NomerOe}&wit=ICKATALOGWEB");
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.34 Safari/537.36");
SetCookies(req);
req.RowRequest.ContentType = "application/x-www-form-urlencoded";
LinkParser link = new LinkParser(req.Request);
List <string> numbers = link.Data.ParsRegex("> ([0-9]+)", 1);
for (int i = 0; i < numbers.Count; i++)
{
if (i == 0)
{
profile.OriginalNumbers = numbers[i];
}
else
{
profile.OriginalNumbers += ";\n" + numbers[i];
}
}
}
public void Should_replace_link_tags()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
parser.ParsedText.ShouldEqual("some text some more text and yet some more");
links.Length.ShouldEqual(2);
}
private void SetModel(IntercarsProfile profile, string data)
{
string numerOe = data.ParsFromTo("daj_numeryOE('", "'");
profile.NomerOe = numerOe;
if (numerOe == "")
{
return;
}
ReqParametres req = new ReqParametres($"https://ic-ua.intercars.eu/dynamic/ickatalogweb/ws_getsoap.php?call=stos&art={numerOe}&wit=ICKATALOGWEB", HttpMethod.POST, $"call=stos&art={numerOe}&wit=ICKATALOGWEB");
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.34 Safari/537.36");
SetCookies(req);
req.RowRequest.ContentType = "application/x-www-form-urlencoded";
LinkParser link = new LinkParser(req.Request);
profile.Mark = link.Data.ParsFromTo("class=\"dZB1\" ><u>", "<");
List <string> models = link.Data.Replace("\n", "").Replace("\r", "").ParsRegex("class=\"dZB2\"><u>(.*?)<", 1);
for (int i = 0; i < models.Count; i++)
{
if (i == 0)
{
profile.Model = models[i];
}
else
{
profile.Model += ";\n" + models[i];
}
}
}
public void TestLinkParser()
{
Dictionary <string, string> index = new Dictionary <string, string>
{
{ "link", "href" },
};
string input = "a@'link'@invalid";
string output = LinkParser.ResolveText((s) =>
{
string item;
if (index.TryGetValue(s, out item))
{
return(item);
}
return(null);
}, input, s => s);
Assert.Equal("ahref@invalid", output);
input = @"a@link @'link' @""link""";
output = LinkParser.ResolveText((s) =>
{
string item;
if (index.TryGetValue(s, out item))
{
return(item);
}
return(null);
}, input, s => "[link](" + s + ")");
Assert.Equal("a[link](href) [link](href) [link](href)", output);
}
private void SetZaminniki(IntercarsProfile profile, string data)
{
string parametres = data.ParsFromTo("&sta=T&fir=UJ8&gru=", "\"");
profile.Gru = parametres;
if (parametres == "")
{
return;
}
ReqParametres req = new ReqParametres($"https://ic-ua.intercars.eu/dynamic/ickatalogweb/ws_zamienniki.php?popup=T&firgru={parametres}&towkod={profile.TovarCode}&zakres=all");
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.34 Safari/537.36");
SetCookies(req);
req.RowRequest.ContentType = "application/x-www-form-urlencoded";
LinkParser link = new LinkParser(req.Request);
List <string> zaminniki = link.Data.ParsRegex(" <b>(.*?)<", 1);
for (int i = 0; i < zaminniki.Count; i++)
{
if (i == 0)
{
profile.Zaminniki = zaminniki[i];
}
else
{
profile.Zaminniki += ";\n" + zaminniki[i];
}
}
}
private void setInfoText(string text)
{
try
{
int previousHeight = linkLabelInfo.Height;
var parser = new LinkParser(text);
var links = parser.Parse();
if (parser.ParsedText.Length > GDI_SIZE_LIMIT)
{
linkLabelInfo.Text = parser.ParsedText.Substring(0, GDI_SIZE_LIMIT);
}
else
{
linkLabelInfo.Text = parser.ParsedText;
}
linkLabelInfo.LinkArea = new LinkArea(0, 0);
foreach (var link in links)
{
if (link.Start + link.Length <= GDI_SIZE_LIMIT)
{
linkLabelInfo.Links.Add(link.Start, link.Length);
}
}
var difference = linkLabelInfo.Height - previousHeight;
Height = Height + difference;
}
catch (Exception exception)
{
_bus.Publish(new ErrorMessage(exception));
}
}
public TelegramChecker(string url, int reconnects)
{
triesToConnect = reconnects;
Url = url;
LinkParser linkParser = null;
do
{
ReqParametres req = new ReqParametres(url);
_currentProxy = Proxies.GetProxy();
req.SetProxy(_currentProxy);
req.SetTimout(8000);
linkParser = new LinkParser(req.Request);
}while (triesToConnect-- > 0 && linkParser.IsError);
if (!linkParser.IsError)
{
string title = linkParser.Data.ParsFromTo("<title>", "</title>");
string data = linkParser.Data.ParsFromTo("<div class=\"tgme_page_extra\">", "</div>");
SetType(title, data);
SetMembers(data);
Proxies.ReturnProxie(_currentProxy);
}
else
{
IsError = true;
}
}
private string UploadImageToGoogle()
{
try
{
Console.Write("Trying connection to new proxy: ");
ReqParametres req = new ReqParametres($"https://www.google.com/searchbyimage?image_url={_imageUrl}");
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.53 Safari/537.36");
//req.SetProxy(5000, Proxies.GetProxy());
LinkParser link = new LinkParser(req.Request);
if (link.IsError)
{
Proxies.DeleteFirstProxy();
return(UploadImageToGoogle());
}
if (link.Data.ParsFromTo("role=\"heading\"><a href=\"", "\"") == "")
{
Proxies.DeleteFirstProxy();
return(UploadImageToGoogle());
}
Console.WriteLine("Succes!");
return("https://www.google.com" + link.Data.ParsFromTo("role=\"heading\"><a href=\"", "\"").Replace("amp;", ""));
}
catch (Exception e)
{
Console.WriteLine(e);
Proxies.DeleteFirstProxy();
return(UploadImageToGoogle());
}
}
private void CheckTrust()
{
LinkParser linkParser;
do
{
ReqParametres reqParametres = new ReqParametres($"https://checktrust.ru/app.php?r=host/app/summary/basic&applicationKey=619f496e644ba8edcd702704176a7b26&host={_site.Link}¶meterList=spam,mjDin,mjCF,mjTF");
linkParser = new LinkParser(reqParametres.Request);
try
{
_site.Bl = decimal.Parse(linkParser.Data.ParsFromTo("mjDin\":\"", "\""), CultureInfo.InvariantCulture);
_site.TrustFlow = decimal.Parse(linkParser.Data.ParsFromTo("mjTF\":\"", "\""), CultureInfo.InvariantCulture);
_site.CitationFlow = decimal.Parse(linkParser.Data.ParsFromTo("mjCF\":\"", "\""), CultureInfo.InvariantCulture);
}
catch (Exception e)
{
Console.WriteLine(e);
}
if (linkParser.Data.Contains("message\":\"Host "))
{
Thread.Sleep(TimeSpan.FromSeconds(5));
}
} while (linkParser.Data.Contains("message\":\"Host "));
}
static void Main(string[] args)
{
var parser = new LinkParser();
var domains = new string[]
{
"https://mail.ru/"
};
var strategies = new ILinkParserStrategy[]
{
new BaseParserStrategy(),
new ExcludeValueParserStrategie("profile"),
new ImageSrcParserStrategy(),
new NotLessThenPageSizeParserStrategy(1024 * 200)
};
Console.CancelKeyPress += new ConsoleCancelEventHandler((object sender, ConsoleCancelEventArgs cancelArgs) =>
{
parser.Stop();
cancelArgs.Cancel = true;
});
parser.Parse(domains, "result.txt", strategies, new ProgressReporter());
Console.WriteLine("Parsing is over");
Console.ReadLine();
}
public override void CrawlData()
{
string baid = "-1";
for (int i = 0; i < MaxPageNumber; i++)
{
try
{
LinkParser linkParser;
do
{
ReqParametres reqParametres = new ReqParametres("https://ru.auctions.godaddy.com/trpSearchResults.aspx", HttpMethod.Post, $"t=5&action=search&hidAdvSearch=ddlAdvKeyword:3|txtKeyword:|ddlCharacters:0|txtCharacters:|txtMinTraffic:|txtMaxTraffic:|txtMinDomainAge:|txtMaxDomainAge:|txtMinPrice:|txtMaxPrice:|ddlCategories:0|chkAddBuyNow:false|chkAddFeatured:false|chkAddDash:true|chkAddDigit:true|chkAddWeb:false|chkAddAppr:false|chkAddInv:false|chkAddReseller:false|ddlPattern1:|ddlPattern2:|ddlPattern3:|ddlPattern4:|chkSaleOffer:false|chkSalePublic:false|chkSaleExpired:false|chkSaleCloseouts:false|chkSaleUsed:false|chkSaleBuyNow:false|chkSaleDC:false|chkAddOnSale:false|ddlAdvBids:0|txtBids:|txtAuctionID:|ddlDateOffset:|ddlSort:auctionendtimeD&rtr=7&baid={baid}&searchDir=1");
reqParametres.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.38 Safari/537.36");
reqParametres.RowRequest.ContentType = "application/x-www-form-urlencoded";
linkParser = new LinkParser(reqParametres.Request);
} while (!linkParser.Data.Contains("onclick=\"_s_baid"));
Sites.AddRange(linkParser.Data.ParsRegex("alt=\"\" />(.*?)<", 1).Select(x => new Site {
Link = x
}));
baid = linkParser.Data.ParsFromTo("onclick=\"_s_baid1=", ";");
if (string.IsNullOrEmpty(baid))
{
break;
}
}
catch (Exception e)
{
Thread.Sleep(5000);
Console.WriteLine(e.Message + "\n");
}
}
}
private void ParsInfo(Zillow zillow)
{
Thread thread = new Thread(() =>
{
LinkParser linkParser;
ReqParametres reqParametres;
do
{
//Парсим предварительную ссылку
reqParametres = new ReqParametres(zillow.URL);
reqParametres.SetUserAgent(Useragents.GetNewUseragent());
reqParametres.SetProxy();
linkParser = new LinkParser(reqParametres.Request);
SavedCookies = linkParser.Cookies;
} while (isCaptcha(linkParser.Data));
string newLink = linkParser.Data.ParsFromTo("<link rel=\"canonical\" href=\"", "\"");
//Проверяем на неверную ссылку
if (newLink.Contains("https://www.zillow.com/homes/for_sale/"))
{
zillow.Status = "No such adress";
}
else
{
zillow.URL = newLink;
do
{
reqParametres = new ReqParametres(zillow.URL);
reqParametres.SetUserAgent(Useragents.GetNewUseragent());
reqParametres.SetProxy();
linkParser = new LinkParser(reqParametres.Request);
} while (isCaptcha(linkParser.Data));
zillow.Status = CheckOnStatus(linkParser.Data.ToLower()).Replace("<span tabindex=\"0\" role=\"button\"><span class=\"zsg-tooltip-launch_keyword\">", "")
.Replace("<Span Tabindex=\"0\" Role=\"Button\"><Span Class=\"Zsg-Tooltip-Launch_Keyword\">", "");
if (zillow.Status.Equals("Undefined"))
{
DebugBox.WriteLine(linkParser.Data);
}
else
{
//Zestimate set
List <string> rawZestimate = linkParser.Data.ParsRegex("Zestimate<sup>®</sup></span></span>(.*?)\\$([0-9,./a-zA-Z]+)<", 2);
if (rawZestimate.Count != 0)
{
zillow.Zestimate = "$" + rawZestimate[0];
}
zillow.SoldPrice = CheckPrice(linkParser.Data);
}
SavedCookies = linkParser.Cookies;
}
threadCount--;
progress++;
DebugBox.WriteLine($"Обработано ссылок: {progress} из {results.Count}.");
double val = 100.0f / results.Count * progress;
WorkProgress.SetValue(val);
});
thread.IsBackground = true;
thread.Start();
}
public static Dictionary <string, string> Find(string body, string url, List <string> userNames, List <DomainData> socialDomains, bool returnOnlyNone200 = true)
{
Dictionary <string, string> foundUrls = new Dictionary <string, string>();
LinkParser parser = new LinkParser();
parser.ParseLinksAgility(body, url, true);
foreach (Request foundUrl in parser.GoodUrls)
{
string foundURL = DomainUtility.StripProtocol(foundUrl.Url.Split('?')[0]);
if (SocialDomainUtility.CheckIfSocialMediaSite(foundURL, socialDomains))
{
if (userNames.Count == 0)
{
if (!foundUrls.ContainsKey(foundURL))
{
Request request = new Request(DomainUtility.EnsureHTTPS(foundURL));
RequestUtility.GetWebText(request);
if (!request.Response.Code.Equals("200") || request.Url.Contains("buymethat"))
{
foundUrls.Add(foundURL, url);
}
else if (!returnOnlyNone200)
{
foundUrls.Add(foundURL, url);
}
}
}
else
{
foreach (string userName in userNames)
{
if (foundURL.ToLower().Contains(userName.ToLower()))
{
if (!foundUrls.ContainsKey(foundURL))
{
if (Ignore.Contains(foundURL.ToLower()))
{
continue;
}
Request request = new Request(DomainUtility.EnsureHTTPS(foundURL));
RequestUtility.GetWebText(request);
if (!request.Response.Code.Equals("200") || request.Url.Contains("buymethat"))
{
foundUrls.Add(foundURL, url);
}
else if (!returnOnlyNone200)
{
foundUrls.Add(foundURL, url);
}
}
}
}
}
}
}
return(foundUrls);
}
public async Task BuildSite(Site site)
{
foreach (var version in site.Versions)
{
// compose doc sections
foreach (var section in site.GetSectionsByVersion(version))
{
var uiBundleRef = LinkParser.Parse("xref://ui-bundle:tanka-docs-section.yml").Xref !.Value;
var uiContent = site.GetSectionByXref(uiBundleRef, section);
if (uiContent == null)
{
throw new InvalidOperationException($"Could not resolve ui-bundle. Xref '{uiBundleRef}' could not be resolved.'");
}
var uiBundle = new HandlebarsUiBundle(site, uiContent, _output);
await uiBundle.Initialize(CancellationToken.None);
var composer = new SectionComposer(site, _cache, _output, uiBundle);
await composer.ComposeSection(section);
}
}
await ComposeIndexPage(site);
}
public void Should_replace_link_tags()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
parser.ParsedText.ShouldEqual("some text some more text and yet some more");
links.Length.ShouldEqual(2);
}
public void Should_return_links()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
links.Length.ShouldEqual(2);
parser.ParsedText.Substring(links[0].Start, links[0].Length).ShouldEqual("some more text");
parser.ParsedText.Substring(links[1].Start, links[1].Length).ShouldEqual("some more");
}
public void Should_return_links()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
links.Length.ShouldEqual(2);
parser.ParsedText.Substring(links[0].Start, links[0].Length).ShouldEqual("some more text");
parser.ParsedText.Substring(links[1].Start, links[1].Length).ShouldEqual("some more");
}
private void should_grab_links()
{
var input = File.ReadAllText(_testFilePath);
var linkGrabber = new LinkParser();
var result = linkGrabber.Parse(input);
Approvals.VerifyAll(result, "[name, link]");
}
public async Task <IActionResult> Index(int page)
{
var houses = await httpWrapper.GetAsync <List <House> >(RelativePaths.HousesEndpoint, page);
var navigationControls = LinkParser.Parse(houses.LinkHeader);
ViewBag.controls = new NavigationControls(navigationControls);
return(View(houses.HttpResponse));
}
public IActionResult AddRelatedProduct(
string id, string pathToRelatedProduct, [FromServices] LinkParser linkParser)
{
var routeValues = linkParser.ParsePathByEndpointName(
nameof(GetProduct), pathToRelatedProduct);
var relatedProductId = routeValues?["id"];
// ...
// </snippet_AddRelatedProduct>
return(NoContent());
}
public void ParseUri()
{
/* Given */
var link = ExternalLink;
/* When */
var definition = LinkParser.Parse(link);
/* Then */
Assert.True(definition.IsExternal);
Assert.False(definition.IsXref);
Assert.Equal("https://link.invalid", definition.Uri);
}
public void Should_return_links()
{
var parser = new LinkParser("some text <<Link>>some more text<</Link>> and yet <<Link>>some more<</Link>>");
var links = parser.Parse();
links.Length.ShouldEqual(2);
// some more text
links[0].Start.ShouldEqual(10);
links[0].Length.ShouldEqual(14);
// some more
links[1].Start.ShouldEqual(33);
links[1].Length.ShouldEqual(9);
}
public void TestLinkParser()
{
Dictionary <string, string> index = new Dictionary <string, string>
{
{ "link", "href" },
};
string input = "a@T:link-@invalid";
string output = LinkParser.ResolveText(index, input, s => s);
Assert.AreEqual("ahref@invalid", output);
input = "a@link @T:link-";
output = LinkParser.ResolveText(index, input, s => "[link](" + s + ")");
Assert.AreEqual("a[link](href) [link](href)", output);
}
public object?ReadYaml(IParser parser, Type type)
{
// should be string
var value = parser.Consume <Scalar>().Value;
if (string.IsNullOrEmpty(value))
{
return(null);
}
var link = LinkParser.Parse(value);
return(link);
}
public void ParseXref_with_SectionId()
{
/* Given */
var link = XrefWithSectionIdLink;
/* When */
var definition = LinkParser.Parse(link);
/* Then */
Assert.True(definition.IsXref);
Assert.False(definition.IsExternal);
Assert.Equal("link.md", definition.Xref?.Path);
Assert.Equal("section", definition.Xref?.SectionId);
}
// Checks URL text box for a video or playlist link and adds the videos to the queue
private void buttonAddVideos_Click(object sender, RoutedEventArgs e)
{
labelUrlStatus.Content = "Loading...";
labelUrlStatus.Refresh();
string videoID = LinkParser.GetVideoIDFromURL(textBoxURL.Text);
string playlistID = LinkParser.GetPlaylistIDFromURL(textBoxURL.Text);
// Check which IDs were found
if (videoID != null && playlistID != null)
{
// Ask if user would like to load the attached playlist
var result = MessageBox.Show(
"This link contains a playlist. Would you like to load the playlist?",
"Playlist Detected",
MessageBoxButton.YesNo);
if (result == MessageBoxResult.Yes)
{
// Loads playlist instead of video
videoID = null;
}
}
if (videoID != null)
{
// Get video and add to list
var video = YouTubeApiService.GetVideo(videoID);
var videoListEntry = new VideoListEntry(video);
VideoList.Add(videoListEntry);
labelUrlStatus.Content = $"\"{video.Title}\" successfully added!";
}
else if (playlistID != null)
{
// Get playlist and add all videos to list
var videos = YouTubeApiService.GetPlaylist(playlistID);
foreach (var video in videos)
{
var videoListEntry = new VideoListEntry(video);
VideoList.Add(videoListEntry);
}
labelUrlStatus.Content = $"Playlist successfully added!";
}
else
{
labelUrlStatus.Content = "Invalid link!";
}
}
private void GetAllGoogleImageLinks(string googleUrl)
{
for (int i = 0; i < 10000; i += 100)
{
try
{
Console.WriteLine($"Trying to get page {i/100} of Google search results.");
ReqParametres req;
if (i == 0)
{
req = new ReqParametres($"{googleUrl}&async=_id:rg_s,_pms:s,_jsfs:Ffpdje,_fmt:pc&asearch=ichunk");
}
else
{
req = new ReqParametres($"{googleUrl}&start={i}&ijn={i / 100}&async=_id:rg_s,_pms:s,_jsfs:Ffpdje,_fmt:pc&asearch=ichunk");
}
req.SetUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.53 Safari/537.36");
//req.SetProxy(3000, Proxies.GetProxy());
LinkParser linkParser = new LinkParser(req.Request);
if (linkParser.IsError)
{
throw new Exception();
}
List <string> allLinksOnSite = linkParser.Data.ParsRegex("\"ou\":\"(.*?)\"", 1);
if (allLinksOnSite.Count == 0)
{
break;
}
allLinksOnSite.ForEach(
x =>
{
if (x.ToLower().EndsWith(".jpg") || x.ToLower().EndsWith(".jpeg") ||
x.ToLower().EndsWith(".png"))
{
CrowledImageLinks.Add(x);
}
});
Console.WriteLine($"{_imagePath}.....Found {CrowledImageLinks.Count} images.");
}
catch (Exception e)
{
Console.WriteLine("Failed.");
Proxies.DeleteFirstProxy();
i -= 100;
continue;
}
}
}
public ContentBlockModel(string providerName, string content, Guid sharedContentId, Type containerType, string wrapperCssClass)
{
this.ContainerType = containerType;
this.ProviderName = providerName;
this.SharedContentID = sharedContentId;
this.WrapperCssClass = wrapperCssClass;
content = this.GetContentHtmlValue(content);
this.Content = LinkParser.ResolveLinks(
content,
DynamicLinksParser.GetContentUrl,
null,
SystemManager.IsInlineEditingMode);
}
public void SetText(string text, string caption)
{
Text = caption;
var parser = new LinkParser(text);
var links = parser.Parse();
linkLabel.Text = parser.ParsedText;
linkLabel.LinkArea = new LinkArea(0, 0);
foreach (var link in links)
{
linkLabel.Links.Add(link.Start, link.Length);
}
Height = linkLabel.Height + (linkLabel.Top * 2) + 28;
Width = linkLabel.Width + (linkLabel.Left * 2) + 8;
}
public ContentBlockModel(string providerName, string content, bool enableSocialSharing, Guid sharedContentId, Type containerType)
{
this.ContainerType = containerType;
this.ProviderName = providerName;
this.EnableSocialSharing = enableSocialSharing;
this.SharedContentID = sharedContentId;
content = this.GetContentHtmlValue(content);
this.Content = LinkParser.ResolveLinks(
content,
DynamicLinksParser.GetContentUrl,
null,
SystemManager.IsInlineEditingMode);
}
public ListModel(string items)
{
this.Items = new List <ListItem>();
if (!String.IsNullOrEmpty(items))
{
this.Items.AddRange(ServiceStack.Text.JsonSerializer.DeserializeFromString <List <ListItem> >(items));
//Resolve the links
foreach (var i in this.Items)
{
i.Content = LinkParser.ResolveLinks(i.Content, DynamicLinksParser.GetContentUrl, null, false);
}
}
}
public Parsers()
{
// Create inline parsers
LineBreakParser = new LineBreakParser();
BacktickParser = new BacktickParser();
InlineCodeParser = new InlineCodeParser();
EscapedCharParser = new EscapedCharParser();
EntityParser = new EntityParser();
StrWithEntitiesParser = new StrWithEntitiesParser(this);
EscapedStringParser = new EscapedStringParser(this);
AutolinkParser = new AutolinkParser(this);
AutolinkEmailParser = new AutolinkEmailParser(this);
RawHtmlParser = new RawHtmlParser();
LinkLabelParser = new LinkLabelParser(this);
LinkDestinationParser = new LinkDestinationParser(this);
LinkTitleParser = new LinkTitleParser(this);
LinkReferenceParser = new LinkReferenceParser(this);
LinkParser = new LinkParser(this);
ImageParser = new ImageParser(this);
ImageReferenceParser = new ImageReferenceParser(this);
LinkDefinitionParser = new LinkDefinitionParser(this);
EmphasisParser = new EmphasisParser(this);
InlineParser = new InlineParser(this);
CommonMarkInlineParser = new CommonMarkInlineParser(this);
// Create block parsers
IndentedCodeParser = new IndentedCodeParser();
LazyParagraphContinuationParser = new LazyParagraphContinuationParser();
BlockQuoteParser = new BlockQuoteParser();
ATXHeaderParser = new ATXHeaderParser();
FencedCodeParser = new FencedCodeParser();
HtmlBlockParser = new HtmlBlockParser();
SetExtHeaderParser = new SetExtHeaderParser();
HorizontalRuleParser = new HorizontalRuleParser();
ListParser = new ListParser();
}
public void LinkWithMultipleParameters()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse("</raw/hb>; rel=\"up\"; riaktag=\"foo\"");
Assert.AreEqual(1, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("foo", links[0].RiakTag);
// we don't strip rel="up" out here since there is a riaktag too.
Assert.AreEqual("</raw/hb>; rel=\"up\"; riaktag=\"foo\"",
links.ToString());
}
private void setInfoText(string text)
{
try
{
int previousHeight = linkLabelInfo.Height;
var parser = new LinkParser(text);
var links = parser.Parse();
if (parser.ParsedText.Length > GDI_SIZE_LIMIT)
linkLabelInfo.Text = parser.ParsedText.Substring(0, GDI_SIZE_LIMIT);
else
linkLabelInfo.Text = parser.ParsedText;
linkLabelInfo.LinkArea = new LinkArea(0, 0);
foreach (var link in links)
{
if (link.Start + link.Length <= GDI_SIZE_LIMIT)
linkLabelInfo.Links.Add(link.Start, link.Length);
}
var difference = linkLabelInfo.Height - previousHeight;
Height = Height + difference;
}
catch (Exception exception)
{
_bus.Publish(new ErrorMessage(exception));
}
}
public void LinkOddlyFormedIsParsedAndNormalized()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse(" </raw/hb> ;rel=\"up\" ; riaktag=\"foo\" ;");
Assert.AreEqual(1, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("foo", links[0].RiakTag);
Assert.AreEqual("</raw/hb>; rel=\"up\"; riaktag=\"foo\"",
links.ToString());
}
public void LinkWithUnknownParameters()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse("</raw/hb>; rel=\"up\"; riaktag=\"foo\"; unk=\"value\"");
Assert.AreEqual(1, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("foo", links[0].RiakTag);
Assert.AreEqual("value", links[0].UnknownParameters["unk"]);
// normalizes as unknown -> rel -> riaktag
Assert.AreEqual("</raw/hb>; unk=\"value\"; rel=\"up\"; riaktag=\"foo\"",
links.ToString());
}
public void LinkWithRelationalUri()
{
LinkParser parser = new LinkParser();
LinkCollection links = parser.Parse("</raw/hb>; rel=\"up\", </raw/hb/fourth>; riaktag=\"foo\"");
Assert.AreEqual(2, links.Count);
Assert.AreEqual("/raw/hb", links[0].UriResource);
Assert.AreEqual("up", links[0].Rel);
Assert.AreEqual("</raw/hb>; rel=\"up\"", links[0].ToString());
Assert.AreEqual("/raw/hb/fourth", links[1].UriResource);
Assert.AreEqual("foo", links[1].RiakTag);
Assert.AreEqual("</raw/hb/fourth>; riaktag=\"foo\"", links[1].ToString());
// we strip out the rel="up" link if it is the only thing in the link
Assert.AreEqual("</raw/hb/fourth>; riaktag=\"foo\"",
links.ToString());
}
