public override async Task <MemeInfo> RandomAsync()
{
WebClient wc;
string html;
try
{
wc = new WebClient();
html = await wc.DownloadStringTaskAsync(_uris[UriType.Random]);
}
catch (WebException ex)
{
throw new ServiceOrConnectionException("Could not load the page", ex);
}
IConfiguration config = Configuration.Default;
IBrowsingContext context = BrowsingContext.New(config);
IDocument document = await context.OpenAsync(req => req.Content(html).Address(_baseUrl));
IElement picDiv = document.DocumentElement.QuerySelector(".media-element-wrapper");
IHtmlImageElement img = (IHtmlImageElement)picDiv.QuerySelector(".figure-holder figure img");
IHtmlHeadingElement h = (IHtmlHeadingElement)picDiv.QuerySelector(".content h1");
IElement player = picDiv.QuerySelector(".figure-holder figure player");
if ((player == null && img == null) || h == null)
{
throw new NotFoundException(
"Either \"img\", \"player\" or \"h\" tag could not be found");
}
// Unfortunately on Kwejk random image page there's no link to the
// original image page, so ViewURI = URI
MemeInfo meme;
if (player != null)
{
meme = new MemeInfo
{
ViewURI = player.GetAttribute("source"),
URI = player.GetAttribute("source"),
Alt = string.Empty,
Name = h.TextContent.Trim(),
Type = MediaType.Video
};
}
else
{
meme = new MemeInfo
{
ViewURI = img.Source,
URI = img.Source,
Alt = img.AlternativeText,
Name = h.TextContent.Trim(),
Type = MediaType.Image
};
}
return(meme);
}
private object GetSerial(IHtmlImageElement ele)
{
Log.Debug("Parent Content: {content}", ele.Parent.TextContent);
Log.Debug("Last Ancestor Content: {content}", ele.Ancestors().Last().Text());
var innerHTML = ele.ParentElement.ParentElement.InnerHtml;
return(innerHTML.AsSpan().Slice(c => 1, c => c.Slice(1).IndexOf('\t') + 1).ToString().ToLower());
}
private Inline GenerateInlineImage(IHtmlImageElement node)
{
var inlineContainer = new InlineUIContainer();
var image = GenerateImage(node);
inlineContainer.Child = image;
return(inlineContainer);
}
private Boolean IsAssociatedImage(IHtmlImageElement image)
{
var usemap = image.UseMap;
if (!String.IsNullOrEmpty(usemap))
{
var name = usemap.Has(Symbols.Num) ? "#" + Name : Name;
return usemap.Is(name);
}
return false;
}
Boolean IsAssociatedImage(IHtmlImageElement image)
{
var usemap = image.UseMap;
if (!String.IsNullOrEmpty(usemap))
{
var name = usemap[0] == '#' ? '#' + Name : Name;
return(usemap == name);
}
return(false);
}
Boolean IsAssociatedImage(IHtmlImageElement image)
{
var usemap = image.UseMap;
if (!String.IsNullOrEmpty(usemap))
{
var name = usemap[0] == '#' ? '#' + Name : Name;
return usemap == name;
}
return false;
}
private Boolean IsAssociatedImage(IHtmlImageElement image)
{
var usemap = image.UseMap;
if (!String.IsNullOrEmpty(usemap))
{
var name = usemap.Has(Symbols.Num) ? "#" + Name : Name;
return(usemap.Is(name));
}
return(false);
}
private void VisitImage(IElement htmlElement)
{
IHtmlImageElement imageElement = htmlElement as IHtmlImageElement;
if (imageElement != null && imageElement.HasAttribute("alt"))
{
var alt = imageElement.GetAttribute("alt");
if (!String.IsNullOrEmpty(alt))
{
AppendText(alt);
}
}
}
private Image GenerateImage(IHtmlImageElement node)
{
var image = new Image()
{
Stretch = Stretch.UniformToFill
};
if (Uri.TryCreate(node.Source, UriKind.RelativeOrAbsolute, out Uri src))
{
var bitmap = new BitmapImage(src);
image.Source = bitmap;
}
return(image);
}
static Stack <IHtmlSourceElement> GetSources(this IHtmlImageElement img)
{
var parent = img.ParentElement;
var sources = new Stack <IHtmlSourceElement>();
if (parent != null && parent.LocalName.Is(Tags.Picture))
{
var element = img.PreviousElementSibling as IHtmlSourceElement;
while (element != null)
{
sources.Push(element);
element = element.PreviousElementSibling as IHtmlSourceElement;
}
}
return(sources);
}
Inline Render(IHtmlImageElement element)
{
var f = new Figure();
f.FlowDirection = FlowDirection.LeftToRight;
var container = new BlockUIContainer();
var img = new Image();
img.Stretch = Stretch.None;
var src = new BitmapImage();
src.BeginInit();
src.UriSource = new Uri(url, Sanitize(element.Source));
src.EndInit();
f.Blocks.Add(container);
container.Child = img;
img.Source = src;
return(f);
}
public override async Task <MemeInfo> RandomAsync()
{
WebClientPlus wc;
string html;
try
{
wc = new WebClientPlus();
html = await wc.DownloadStringTaskAsync(_uris[UriType.Random]);
}
catch (WebException ex)
{
throw new ServiceOrConnectionException("Could not load the page", ex);
}
IConfiguration config = Configuration.Default;
IBrowsingContext context = BrowsingContext.New(config);
IDocument document = await context.OpenAsync(req => req.Content(html).Address(_baseUrl));
IElement picDiv = document.DocumentElement.QuerySelector("#main_container .pic");
IHtmlImageElement img =
(IHtmlImageElement)picDiv.QuerySelector(".pic_image img");
IHtmlHeadingElement h =
(IHtmlHeadingElement)picDiv.QuerySelector("h1.picture");
if (img == null || h == null)
{
throw new NotFoundException("Either \"img\" or \"h1\" tag could not be found");
}
return(new MemeInfo
{
ViewURI = wc.ResponseURI.ToString(),
URI = img.Source,
Alt = img.AlternativeText,
Name = h.TextContent
});
}
private FrameworkElement GenerateImage(IHtmlImageElement node)
{
var image = new Image()
{
HorizontalAlignment = Windows.UI.Xaml.HorizontalAlignment.Center
};
var viewBox = new Viewbox()
{
Stretch = Stretch.UniformToFill,
StretchDirection = StretchDirection.DownOnly,
HorizontalAlignment = Windows.UI.Xaml.HorizontalAlignment.Center
};
viewBox.Child = image;
if (Uri.TryCreate(node.Source, UriKind.RelativeOrAbsolute, out Uri src))
{
var bitmap = new BitmapImage(src);
image.Source = bitmap;
}
return(viewBox);
}
public override async Task <MemeInfo> RandomAsync()
{
WebClient wc;
string html;
try
{
wc = new WebClient();
html = await wc.DownloadStringTaskAsync(_uris[UriType.Random]);
}
catch (WebException ex)
{
throw new ServiceOrConnectionException("Could not load the page", ex);
}
IConfiguration config = Configuration.Default;
IBrowsingContext context = BrowsingContext.New(config);
IDocument document = await context.OpenAsync(req => req.Content(html).Address(_baseUrl));
IElement picDiv = document.DocumentElement
.QuerySelectorAll("#wrapper-wrap .left .ob-left-box-images")[RandomNthChild()];
IHtmlImageElement img = (IHtmlImageElement)picDiv
.QuerySelector(".left-wrap a img:last-child");
IHtmlAnchorElement a = (IHtmlAnchorElement)picDiv
.QuerySelector("h2 a");
IHtmlInputElement input = (IHtmlInputElement)picDiv
.QuerySelector(".left-wrap input[type=\"hidden\"]");
IHtmlSourceElement src = (IHtmlSourceElement)picDiv
.QuerySelector(".left-wrap video > source");
if ((src == null && img == null) || a == null)
{
throw new NotFoundException(
"Either \"img\", \"source\" or \"a\" tag could not be found");
}
MemeInfo meme;
if (src != null)
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = src.Source,
Alt = string.Empty,
Name = a.TextContent,
Type = MediaType.Video
};
}
else if (input != null)
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = input.Value,
Alt = img.AlternativeText,
Name = a.TextContent,
Type = MediaType.Gif
};
}
else
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = img.Source,
Alt = img.AlternativeText,
Name = a.TextContent,
Type = MediaType.Image
};
}
return(meme);
}
public static IEnumerable <Func <string, Task <ImageSearchResult> > > GetUrls(string html_document)
{
var browsingContext = BrowsingContext.New(Configuration.Default);
var task = browsingContext.OpenAsync(req => req.Content(html_document));
task.Wait();
var document = task.Result;
//var test = document.QuerySelectorAll("div.rg_meta");
var test = document.QuerySelectorAll("a.rg_l");
if (!test.Any())
{
test = document.QuerySelectorAll("img.rg_i");
}
foreach (var element in test)
{
IHtmlImageElement htmlimage = element as IHtmlImageElement;
if (htmlimage != null && htmlimage.Source.StartsWith("https"))
{
Debug.WriteLine("Element is url");
}
else if (htmlimage != null && htmlimage.Source != "")
{
yield return(new Func <string, Task <ImageSearchResult> >((path) =>
{
return new Task <ImageSearchResult>(() =>
{
return ImageSearchResult.FromBase64(path, htmlimage.Source);
});
}));
}
else if (htmlimage != null)
{
Regex re = new Regex("data-src=\".*?\"");
var match = re.Match(htmlimage.OuterHtml);
if (match.Success)
{
var url = match.Value.Substring("data-src=\"".Length, match.Value.Length - "data-src=\"".Length - 1);
yield return(new Func <string, Task <ImageSearchResult> >((path) =>
{
return new Task <ImageSearchResult>(() =>
{
return ImageSearchResult.FromWebUrl(path, url);
});
}));
}
else
{
throw new Exception("Unhandled data source");
}
}
IHtmlAnchorElement htmlanchor = element as IHtmlAnchorElement;
if (htmlanchor != null)
{
var imglink = htmlanchor.Href.Replace(@"%3A", ":").Replace(@"%2F", @"/");
var match = Regex.Match(imglink, @"imgurl.*?&");
if (match.Success)
{
imglink = match.Value.Replace(@"imgurl=", "").Replace("&", "");
yield return(new Func <string, Task <ImageSearchResult> >((path) =>
{
return new Task <ImageSearchResult>(() =>
{
return ImageSearchResult.FromWebUrl(path, imglink);
});
}));
}
var a = 10;
}
else
{
Debug.WriteLine("Element is not a HTML image");
}
}
}
public ImgNode(IHtmlImageElement content)
{
Content = content;
}
public override async Task <MemeInfo> RandomAsync()
{
PseudoRandomImage image = GetPseudoRandomImage();
WebClient wc;
string html;
try
{
wc = new WebClient();
html = await wc.DownloadStringTaskAsync(image.Site);
}
catch (WebException ex)
{
throw new ServiceOrConnectionException("Could not load the page", ex);
}
IConfiguration config = Configuration.Default;
IBrowsingContext context = BrowsingContext.New(config);
IDocument document = await context.OpenAsync(req => req.Content(html).Address(_baseUrl));
IElement picDiv = document.DocumentElement.QuerySelectorAll(
"#content-container article .article-content")[image.NthChild];
IHtmlImageElement img = (IHtmlImageElement)picDiv
.QuerySelector(".article-image img");
IHtmlAnchorElement a = (IHtmlAnchorElement)picDiv
.QuerySelector(".article-title a");
IHtmlSourceElement src = (IHtmlSourceElement)picDiv
.QuerySelector(".article-image video > source");
if ((img == null && src == null) || a == null)
{
throw new NotFoundException(
"Either \"img\", \"source\" or \"a\" tag could not be found");
}
MemeInfo meme;
if (src != null)
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = src.Source,
Alt = string.Empty,
Name = a.TextContent.Trim(),
Type = MediaType.Video
};
}
else
{
meme = new MemeInfo
{
ViewURI = a.Href,
URI = img.Source,
Alt = img.AlternativeText,
Name = a.TextContent.Trim(),
Type = MediaType.Image
};
}
return(meme);
}
public async Task <IEnumerable <IDeal> > Scrape(CancellationToken token)
{
List <IDeal> deals = new List <IDeal>();
DocumentRequest request = DocumentRequest.Get(Url.Create(URL));
IDocument document = await context.OpenAsync(request, token);
token.ThrowIfCancellationRequested();
IHtmlElement body = document.Body;
IEnumerable <IHtmlListItemElement> items = body.QuerySelectorAll <IHtmlListItemElement>(".grid-tile");
foreach (IHtmlListItemElement element in items)
{
IHtmlDivElement titleElement = element.QuerySelector <IHtmlDivElement>(".card-title");
string title = titleElement.TextContent.Trim();
IHtmlAnchorElement linkElement = element.QuerySelector <IHtmlAnchorElement>(".thumb-link");
IHtmlImageElement imageElement = element.QuerySelector <IHtmlImageElement>(".product_image");
IHtmlDivElement priceElement = element.QuerySelector <IHtmlDivElement>(".card-price");
IHtmlDivElement availabilityElement = element.QuerySelector <IHtmlDivElement>(".product-availability-label");
string offerStartDate = availabilityElement.GetAttribute("data-freeofferstartdate");
string offerEndDate = availabilityElement.GetAttribute("data-freeofferenddate");
bool hasStartDate = DateTime.TryParseExact(offerStartDate, "ddd MMM dd HH:mm:ss Z yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime startDate);
bool hasEndDate = DateTime.TryParseExact(offerEndDate, "ddd MMM dd HH:mm:ss Z yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime endDate);
if (!hasStartDate && !hasEndDate)
{
logger.Info($"{title} has no start or end date");
continue;
}
DateTime now = DateTime.Now;
if ((hasStartDate && now < startDate) || (hasEndDate && now > endDate))
{
logger.Info($"{title} is not active right now");
continue;
}
string price = priceElement.TextContent.Trim();
if (price.ToLower() != "free to play")
{
logger.Info($"{title} is not free");
continue;
}
deals.Add(new Deal
{
Discount = 100,
End = hasEndDate ? endDate : (DateTime?)null,
Start = hasStartDate ? startDate : (DateTime?)null,
Title = title,
Link = $"https://store.ubi.com/{linkElement.GetAttribute("href")}",
Image = imageElement.GetAttribute("data-desktop-src")
});
}
return(deals);
}
private int GetGenderFrom(IHtmlImageElement image)
=> image.Source.Contains("V") ? 1 : 2;
