コード例 #1
0
ファイル: BasicScripting.cs プロジェクト: zzc000/AngleSharp
        public async Task DynamicallyAddedScriptWithSourceShouldBeExecutedAfterAppending()
        {
            var didRun    = false;
            var scripting = new CallbackScriptEngine(options => didRun = true);
            var config    = Configuration.Default.WithScripts(scripting).WithMockRequester();
            var source    = "<title>Some title</title><body>";
            var document  = await BrowsingContext.New(config).OpenAsync(m => m.Content(source).Address("http://www.example.com"));

            var script = document.CreateElement("script");

            script.SetAttribute("type", scripting.Type);
            script.SetAttribute("src", "foo.cs");

            Assert.IsFalse(didRun);

            document.Body.AppendChild(script);

            Assert.IsTrue(didRun);
        }
コード例 #2
0
ファイル: T_StockService.cs プロジェクト: archer0615/MyGit
        public async Task <Stock5DayGainDropViewModel> GetStockGainDrop5Day(string stock_id)
        {
            Stock5DayGainDropViewModel vm       = new Stock5DayGainDropViewModel();
            Stock5DayGainDropSelector  selector = new Stock5DayGainDropSelector();

            string URL = stock5DayGainDrop + stock_id + ".htm";

            var dom = await BrowsingContext.New(config).OpenAsync(URL);

            var mainSelector = @"#main3 > div.mbx.bd3 > div.tab > table > tbody > ";

            var flg = 2;

            for (int tr = 0; tr < 5; tr++)
            {
                selector.Days.Add(new GainDrop());
                vm.Days.Add(new GainDrop());
                selector.Days[tr].GainDropDate = mainSelector + $"tr:nth-child({tr + flg}) > td:nth-child(1)";
                selector.Days[tr].Per          = mainSelector + $"tr:nth-child({tr + flg}) > td:nth-child(7)";
            }

            for (int i = 0; i < vm.Days.Count; i++)
            {
                var newStockProp = TypeDescriptor.GetProperties(selector.Days[i]).Cast <PropertyDescriptor>();
                var setStockProp = TypeDescriptor.GetProperties(vm.Days[i]).Cast <PropertyDescriptor>();

                foreach (var prop in newStockProp)
                {
                    var selectorString = prop.GetValue(selector.Days[i]).ToString();
                    var data           = (Object)dom.QuerySelector(selectorString).TextContent ?? "";

                    foreach (var item in setStockProp)
                    {
                        if (prop.Name == item.Name)
                        {
                            vm.Days[i].GetType().GetProperty(item.Name).SetValue(vm.Days[i], data);
                        }
                    }
                }
            }

            return(vm);
        }
コード例 #3
0
        public async Task GetIndex_PageTitle_IsCorrect()
        {
            // Arrange
            var client = Arrange();

            // Act
            var result = await client.GetAsync("/");

            // Assert
            result.StatusCode.Should().Be(HttpStatusCode.OK);
            var responseStream = await result.Content.ReadAsStreamAsync();

            var context  = BrowsingContext.New(Configuration.Default);
            var document = await context.OpenAsync(req => req.Content(responseStream));

            var title = document.QuerySelector("title");

            title.TextContent.Should().Be("Home Page - MvcSample");
        }
コード例 #4
0
        public List <TEntity> ParseHtmlStringToProducts <TEntity>(List <string> responses) where TEntity : Product, new()
        {
            List <TEntity> _products = new List <TEntity>();
            var            config    = Configuration.Default;
            var            context   = BrowsingContext.New(config);
            var            parser    = context.GetService <AngleSharp.Html.Parser.IHtmlParser>();

            foreach (var res in responses)
            {
                var document = parser.ParseDocument(res);
                var coll     = document.GetElementsByClassName("catalog-taxons-product__hover");
                foreach (var a in coll)
                {
                    try
                    {
                        var product = new TEntity()
                        {
                            Name  = a.GetElementsByClassName("catalog-taxons-product__name").FirstOrDefault().InnerHtml.Trim(),
                            Price = decimal.Parse(MakeDecimalString(a.GetElementsByClassName("catalog-taxons-product-price__item-price").FirstOrDefault().ChildNodes[1].TextContent.Replace(".", ","))),
                            Shop  = (int)ShopName.Shop1A,
                            Url   = "https://www.1a.lv" + a.GetElementsByClassName("catalog-taxons-product__name").FirstOrDefault().GetAttribute("href")
                        };
                        _products.Add(product);
                    }
                    catch
                    {
                        //var product = new TEntity()
                        //{
                        //    Name = a.GetElementsByClassName("catalog-taxons-product__name").FirstOrDefault().InnerHtml.Trim(),
                        //    Price = decimal.Parse(MakeDecimalString(a.GetElementsByClassName("product-price-details__price-number").FirstOrDefault().ChildNodes[1].TextContent.Replace(".", ","))),
                        //    Shop = ShopName.Shop1A,
                        //    Url = "https://www.1a.lv" + a.GetElementsByClassName("catalog-taxons-product__name").FirstOrDefault().GetAttribute("href")
                        //};
                        //_products.Add(product);
                    }
                }
                if (_products.Count == 0)
                {
                    OnZeroProductsParsed(EventArgs.Empty);
                }
            }
            return(_products);
        }
コード例 #5
0
ファイル: Program.cs プロジェクト: Danko19/WikiParser
        private static void DoShit2()
        {
            var i = 1;
            var cookieContainer = new CookieContainer();

            using (var handler = new HttpClientHandler {
                CookieContainer = cookieContainer
            })
            {
                using (var client = new HttpClient(handler))
                {
                    using (var mySql = new MySqlStorage())
                    {
                        foreach (var abbreviation in mySql.GetAll <AbbreviationEntity>().OrderBy(x => x.Decryption?.Length ?? 0))
                        {
                            if (string.IsNullOrEmpty(abbreviation.Decryption))
                            {
                                continue;
                            }
                            var response = client.GetAsync(
                                abbreviation.Link)
                                           .GetAwaiter()
                                           .GetResult();
                            var html     = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
                            var config   = Configuration.Default;
                            var context  = BrowsingContext.New(config);
                            var document = context.OpenAsync(req => req.Content(html)).Result;
                            var div      = document.GetElementsByClassName("mw-parser-output").Single();

                            var ol = div.Children
                                     .SkipWhile(x =>
                                                x.TagName != "H4" ||
                                                !x.Children.Any(z => z.TagName == "SPAN" && z.Id == "Значение"))
                                     .Skip(1)
                                     .FirstOrDefault(x => x.TagName == "OL");

                            abbreviation.Decryption = ol?.TextContent;
                            mySql.InsertOrUpdate(abbreviation);
                        }
                    }
                }
            }
        }
コード例 #6
0
        public void NormalModeShouldError()
        {
            var html    = @"<!DOCTYPE html>
<title>Test</title>
<body>
    <div myAttribute=""blabla>123</div>
</body>";
            var errors  = new List <HtmlErrorEvent>();
            var options = new HtmlParserOptions {
                IsStrictMode = false
            };
            var context = BrowsingContext.New(Configuration.Default);

            context.ParseError += (s, ev) => errors.Add((HtmlErrorEvent)ev);
            var parser = new HtmlParser(options, context);

            parser.Parse(html);
            Assert.AreEqual(1, errors.Count);
        }
コード例 #7
0
        public async Task TogglingSpanIsHiddenCheckedWorksAsExpected()
        {
            var source   = "<span></span>";
            var cfg      = Configuration.Default;
            var document = await BrowsingContext.New(cfg).OpenAsync(res => res.Content(source));

            var span = document.QuerySelector <IHtmlElement>("span");

            Assert.IsFalse(span.IsHidden);

            span.IsHidden = true;

            Assert.IsTrue(span.IsHidden);

            span.IsHidden = false;

            Assert.IsFalse(span.IsHidden);
            Assert.AreEqual(0, span.Attributes.Length);
        }
コード例 #8
0
        public async Task TogglingDetailsIsOpenCheckedWorksAsExpected()
        {
            var source   = "<details></details>";
            var cfg      = Configuration.Default;
            var document = await BrowsingContext.New(cfg).OpenAsync(res => res.Content(source));

            var details = document.QuerySelector <IHtmlDetailsElement>("details");

            Assert.IsFalse(details.IsOpen);

            details.IsOpen = true;

            Assert.IsTrue(details.IsOpen);

            details.IsOpen = false;

            Assert.IsFalse(details.IsOpen);
            Assert.AreEqual(0, details.Attributes.Length);
        }
コード例 #9
0
        public async Task TogglingOptionIsDefaultSelectedWorksAsExpected()
        {
            var source   = "<option></option>";
            var cfg      = Configuration.Default;
            var document = await BrowsingContext.New(cfg).OpenAsync(res => res.Content(source));

            var option = document.QuerySelector <IHtmlOptionElement>("option");

            Assert.IsFalse(option.IsDefaultSelected);

            option.IsDefaultSelected = true;

            Assert.IsTrue(option.IsDefaultSelected);

            option.IsDefaultSelected = false;

            Assert.IsFalse(option.IsDefaultSelected);
            Assert.AreEqual(0, option.Attributes.Length);
        }
コード例 #10
0
        public async Task GoalTest()
        {
            var page = await BrowsingContext.New(Configuration.Default.WithDefaultLoader())
                       .OpenAsync(testAddress);

            var goal_element = page.QuerySelector(".lead");

            Goal goal_values = new Goal()
            {
                zenny_reward     = 9600,
                hrp_reward       = 500,
                wycadpts_reward  = 960,
                goal_description = "Hunt a Glavenus"
            };

            Goal goal_data = await questManager.GetGoal(goal_element);

            goal_values.ShouldDeepEqual(goal_data);
        }
コード例 #11
0
ファイル: Parser.cs プロジェクト: jholm117/recipe.md
        public async Task <RecipeDto> ParseRecipeHtml(Uri uri, DomainSelectors profile, CancellationToken cancellationToken)
        {
            var config   = Configuration.Default.WithDefaultLoader();
            var context  = BrowsingContext.New(config);
            var document = await context.OpenAsync(uri.OriginalString, cancellation : cancellationToken);

            var title       = document.QuerySelector(profile.TitleSelector).TextContent.Trim();
            var ingredients = document.QuerySelectorAll(profile.IngredientSelector).Select(x => x.TextContent.Trim());
            var directions  = document.QuerySelectorAll(profile.DirectionSelector).Select(x => x.TextContent.Trim()).ToList();
            var url         = uri.ToString();

            return(new RecipeDto
            {
                Title = title,
                Ingredients = ingredients,
                Directions = directions,
                Url = url
            });
        }
コード例 #12
0
        public async Task TogglingInputAutoFocusWorksAsExpected()
        {
            var source   = "<input></input>";
            var cfg      = Configuration.Default;
            var document = await BrowsingContext.New(cfg).OpenAsync(res => res.Content(source));

            var input = document.QuerySelector <IHtmlInputElement>("input");

            Assert.IsFalse(input.Autofocus);

            input.Autofocus = true;

            Assert.IsTrue(input.Autofocus);

            input.Autofocus = false;

            Assert.IsFalse(input.Autofocus);
            Assert.AreEqual(0, input.Attributes.Length);
        }
コード例 #13
0
        public static async Task <IList <Article> > ReadPageArticles(string url)
        {
            var articles = new List <Article>();

            var config = Configuration.Default.WithDefaultLoader();

            var document = await BrowsingContext.New(config).OpenAsync(url);

            var articleSelector = ".white-row div.post";

            var cells = document.QuerySelectorAll(articleSelector);

            foreach (var cell in cells)
            {
                articles.Add(ReadArticle(cell));
            }

            return(articles);
        }
コード例 #14
0
    void Start()
    {
        IConfiguration config = Configuration.Default.WithDefaultLoader();
        string         url    = "https://wear.tw";
        IDocument      doc    = BrowsingContext.New(config).OpenAsync(url).Result;

        /*CSS Selector寫法*/
        //IHtmlCollection<IElement> imgs = doc.QuerySelectorAll("ul li div.image_container p.img a.over img");//取得圖片
        IHtmlCollection <IElement> imgs = doc.QuerySelectorAll("ul > li.large > div.image_container > p.img > a.over > img");//取得圖片

        foreach (IElement img in imgs)
        {
            URl = "https:" + img.GetAttribute("data-original");
            Debug.Log(URl);


            StartCoroutine(LoadPic(URl, path));
        }
    }
コード例 #15
0
        public async Task ContextNavigateFromLinkToPage()
        {
            if (Helper.IsNetworkAvailable())
            {
                var title    = "PostUrlencodeNormal";
                var address  = "http://anglesharp.azurewebsites.net/";
                var config   = new Configuration().WithDefaultLoader();
                var context  = BrowsingContext.New(config);
                var document = await context.OpenAsync(address);

                var anchors = document.QuerySelectorAll <IHtmlAnchorElement>("ul a");
                var anchor  = anchors.Where(m => m.TextContent == title).FirstOrDefault();
                var result  = await anchor.Navigate();

                Assert.IsNotNull(result);
                Assert.AreEqual(result, context.Active);
                Assert.AreEqual(title, context.Active.Title);
            }
        }
コード例 #16
0
            internal static async Task <HtmlDocumentResponse> Create([NotNull] StreamResponse streamResponse)
            {
                if (streamResponse == null)
                {
                    throw new ArgumentNullException(nameof(streamResponse));
                }

                IBrowsingContext context = BrowsingContext.New();

                try {
                    IDocument document = await context.OpenAsync(req => req.Content(streamResponse.Content, true)).ConfigureAwait(false);

                    return(new HtmlDocumentResponse(streamResponse, document));
                } catch (Exception e) {
                    ASF.ArchiLogger.LogGenericWarningException(e);

                    return(null);
                }
            }
コード例 #17
0
        public async Task GetTopLevelView_WithoutAuthorization_ReturnsSuccess(string path, string title)
        {
            await RunAsync(async factory =>
            {
                var client = factory.CreateClient();

                var response = await client.GetAsync(path);

                response.EnsureSuccessStatusCode();

                var content = await response.Content.ReadAsStringAsync();

                var context  = BrowsingContext.New(Configuration.Default);
                var document = await context.OpenAsync(req => req.Content(content));

                Assert.Equal(HttpStatusCode.OK, response.StatusCode);
                Assert.Equal(title + " - CardHero.NetCoreApp.TypeScript", document.Title);
            });
        }
コード例 #18
0
        private async Task <IEnumerable <Review> > GetPageReviews(string asin, int pageNum, CancellationToken cancellationToken)
        {
            var url = $"https://www.amazon.com/product-reviews/{asin}/ref=cm_cr_arp_d_viewopt_srt?sortBy=recent&pageNumber={pageNum}";

            var config   = Configuration.Default.WithDefaultLoader();
            var context  = BrowsingContext.New(config);
            var document = await context.OpenAsync(url, cancellationToken);

            return(document
                   .QuerySelectorAll("[data-hook=\"review\"]")
                   .Select(r => new Review
            {
                Asin = asin,
                Rating = r.GetReviewRating(),
                ReviewContent = r.GetReviewContent(),
                ReviewDate = r.GetReviewDate(),
                ReviewTitle = r.GetReviewTitle()
            }));
        }
コード例 #19
0
        public async Task VisitRootPage_ShouldRenderTwoMateriaCardsAndTheFirstOneMustHaveCertainCardSubtitle()
        {
            // Arrange
            var response = await _client.GetAsync("/");

            /// Create a new context for evaluating webpages with the given config
            var context = BrowsingContext.New(Configuration.Default);

            // Act
            var pageContent = await response.Content.ReadAsStringAsync();

            var parsedDocument = context.GetService <IHtmlParser>().ParseDocument(pageContent);

            // Assert
            response.EnsureSuccessStatusCode();
            Assert.Equal(actual: parsedDocument.QuerySelectorAll(".card").Length, expected: 2);
            Assert.Contains(actualString: parsedDocument.QuerySelectorAll(".card-subtitle")[0].TextContent
                            , expectedSubstring: "Horas Semanales: 4");
        }
コード例 #20
0
        public async Task AddAndInvokeClickHandlerWithStringFunctionWontWork()
        {
            var service  = new JavaScriptProvider();
            var cfg      = Configuration.Default.With(service);
            var html     = @"<!doctype html>
<html>
<body>
<script>
var clicked = false;
document.onclick = 'clicked = true;';
document.onclick();
</script>
</body>";
            var document = await BrowsingContext.New(cfg).OpenAsync(m => m.Content(html));

            var clicked = service.Engine.GetOrCreateJint(document).GetValue("clicked").AsBoolean();

            Assert.IsFalse(clicked);
        }
コード例 #21
0
        public static async Task <News[]> ParseNews(string url)
        {
            var config   = Configuration.Default.WithDefaultLoader();
            var document = await BrowsingContext.New(config).OpenAsync(url);

            List <News> news = new List <News>();

            var titlePostList   = document.QuerySelectorAll("h3[data-region-content=forum-post-core-subject]");
            var authorPostList  = document.QuerySelectorAll("h3[data-region-content=forum-post-core-subject]+address a");
            var contentPostList = document.QuerySelectorAll(".post-content-container");

            for (int i = 0; i < titlePostList.Length; i++)
            {
                news.Add(new News(titlePostList[i].TextContent, authorPostList[i].TextContent, "",
                                  new StringBuilder(contentPostList[i].TextContent)));
            }

            return(news.ToArray());
        }
コード例 #22
0
        public async Task <Item> GetItem(string address)
        {
            try {
                var page = await BrowsingContext.New(Configuration.Default.WithDefaultLoader()).OpenAsync(address);

                string name        = page.QuerySelector("h3[itemprop=\"name\"]").TextContent;
                var    itemIntData = page.QuerySelectorAll("div.lead");
                string combination;
                var    combinationDiv = page.QuerySelector(".col-lg-12 div");
                if (combinationDiv != null)
                {
                    var comboitems = combinationDiv.QuerySelectorAll("a").OfType <IHtmlAnchorElement>().ToArray();
                    try {
                        combination = comboitems[0].TextContent + " + " + comboitems[1].TextContent;
                    }
                    catch (IndexOutOfRangeException) {
                        combination = "None";
                    }
                }
                else
                {
                    combination = "None";
                }
                Console.WriteLine("Retrieved " + name.Trim() + ".");
                return(new Item()
                {
                    item_name = name, // name
                    description = page.QuerySelector("p[itemprop=\"description\"]").TextContent,
                    rarity = itemIntData[0].TextContent.ToInt(),
                    max_stack = itemIntData[1].TextContent.ToInt(),
                    sell_price = itemIntData[2].TextContent.ToInt(),
                    combinations = combination
                });
            }
            catch (Exception ex) {
                Console.WriteLine(ex.ToString());
                Console.ForegroundColor = ConsoleColor.Red;
                Console.WriteLine("Error on page " + address + ". Retrying.");
                Console.ResetColor();
                return(await GetItem(address));
            }
        }
コード例 #23
0
        private async Task <List <ProductCreateDto> > GetNewData(string url, IDocument document)
        {
            if (document == null)
            {
                throw new ArgumentNullException(nameof(document));
            }

            var config   = Configuration.Default.WithDefaultLoader();
            var context  = BrowsingContext.New(config);
            var items    = document.QuerySelectorAll("div.fs-product-card").Select(m => m.OuterHtml);
            var products = new List <ProductCreateDto>();
            var category = (url.Split("/").Last()).Split('?').First();

            foreach (var item in items)
            {
                var itemHtml = await context.OpenAsync(req => req.Content(item));

                var product = itemHtml.QuerySelector("div.js-product-card-footer").GetAttribute("data-options");

                using var doc = JsonDocument.Parse(product);
                var root    = doc.RootElement;
                var details = root.GetProperty("ProductDetails");

                var productItem = new ProductCreateDto
                {
                    ProductId = root.GetProperty("productId").ToString(),
                    Supplier  = "ParknSave",
                    Category  = category,
                    Name      = root.GetProperty("productName").ToString(),
                    Img       = itemHtml.QuerySelector("div.fs-product-card__product-image").GetAttribute("data-src-s"),
                    Prefix    = details.GetProperty("MultiBuyDeal").ToString(),
                    Price     = details.GetProperty("PricePerItem").ToString(),
                    Unit      = details.GetProperty("PriceMode").ToString(),
                    Compare   = 1,
                    Latest    = true,
                    Date      = DateTime.Now
                };
                products.Add(productItem);
            }

            return(products);
        }
コード例 #24
0
        private static void Demo()
        {
            // 设置配置以支持文档加载
            var    config   = Configuration.Default.WithDefaultLoader();
            string mainSite = "https://www.cnblogs.com/";

            // 请求博客园首页
            var mainSiteDocument = BrowsingContext.New(config).OpenAsync(mainSite);

            var lastPageSelector = mainSiteDocument.Result.QuerySelector("#paging_block > .pager > a.last");
            var totlaPage        = 0;

            if (null != lastPageSelector)
            {
                Int32.TryParse(lastPageSelector.TextContent, out totlaPage);
            }

            if (totlaPage == 0)
            {
                return;
            }

            for (var i = 1; i <= totlaPage; i++)
            {
                var eachPageUrl      = $"https://www.cnblogs.com/#p{i}";
                var eachPageDocument = BrowsingContext.New(config).OpenAsync(eachPageUrl);

                var itemSelector = eachPageDocument.Result.QuerySelectorAll("#post_list > .post_item");
                foreach (var item in itemSelector)
                {
                    CNBlog cNBlog = GetCNBlog(item);
                    if (null == cNBlog)
                    {
                        continue;
                    }

                    string message = Newtonsoft.Json.JsonConvert.SerializeObject(cNBlog);
                    Console.WriteLine(message);
                    RabbitUtil.Send(message);
                }
            }
        }
コード例 #25
0
        /// <exception cref="DeprecatedWrapperException"></exception>
        internal async Task <IEnumerable <HitAndRunTorrent> > ExtractResultsFromAsync(string html)
        {
            List <HitAndRunTorrent> torrents = new List <HitAndRunTorrent>();

            var context  = BrowsingContext.New(Configuration.Default);
            var document = await context.OpenAsync(req => req.Content(html)).ConfigureAwait(false);

            IElement container = document.QuerySelector("div.lista_all div.box_torrent_all");

            if (container == null)
            {
                throw new DeprecatedWrapperException("Couldn't find the torrent container!", document.DocumentElement);
            }

            IHtmlCollection <IElement> torrentNodes = container.QuerySelectorAll("div.hnr_torrents");

            if (torrentNodes == null)
            {
                throw new DeprecatedWrapperException("Couldn't find the Hit-And-Run torrent nodes!", container);
            }

            if (NoHitAndRun(torrentNodes))
            {
                return(torrents);
            }

            DateTime currentTime = DateTime.Now;

            foreach (IElement torrentNode in torrentNodes)
            {
                try
                {
                    torrents.Add(ParseAsTorrent(torrentNode, currentTime));
                }
                catch (Exception ex)
                {
                    throw new DeprecatedWrapperException("Couldn't parse torrent details!", torrentNode, ex);
                }
            }

            return(torrents);
        }
コード例 #26
0
        public async override Task Run()
        {
            // We require a custom configuration with JavaScript
            var config = Configuration.Default
                         .WithJs()
                         .WithConsoleLogger(context => new StandardConsoleLogger());

            // This is our sample source, we will trigger the load event
            var source   = @"<!doctype html>
                                <html>
                                <head><title>Custom Event sample</title></head>
                                <body>
                                <script>
                                console.log('Before setting the handler!');

                                document.addEventListener('load', function() {
                                    console.log('Document loaded!');
                                });

                                document.addEventListener('hello', function() {
                                    console.log('hello world from JavaScript!');
                                });

                                console.log('After setting the handler!');
                                </script>
                                </body>";
            var document = await BrowsingContext.New(config).OpenAsync(m => m.Content(source));

            // HTML should be output in the end
            Console.WriteLine(document.DocumentElement.OuterHtml);

            // Register Hello event listener from C# (we also have one in JS)
            document.AddEventListener("hello", (s, ev) =>
            {
                Console.WriteLine("hello world from C#!");
            });

            var e = document.CreateEvent("event");

            e.Init("hello", false, false);
            document.Dispatch(e);
        }
コード例 #27
0
        // This method returns all products information
        private async Task <ICollection <NewProductModel> > GetProductsDescriptions(IHtmlCollection <IElement> startPageElement, string startPageAddress, IConfiguration config)
        {
            Random random = new Random();
            ICollection <NewProductModel> models = new List <NewProductModel>();

            startPageAddress = startPageAddress.Remove(startPageAddress.Length - 1);
            foreach (var productLinkElemment in startPageElement)
            {
                // Every product page is loaded with random delay(1-10 sec) to prevent blocking site access or more important - site crashing.
                Thread.Sleep(random.Next(1000, 10000));
                decimal productPrice       = 0;
                string  productDescription = null;
                byte[]  productImg         = null;
                string  productName        = productLinkElemment.TextContent;
                // Extract product page link from anchor's href attribute
                string link = startPageAddress + productLinkElemment.Attributes.Single(attr => attr.Name == "href")?.Value;
                if (!string.IsNullOrEmpty(link))
                {
                    IDocument productPageElement = await BrowsingContext.New(config).OpenAsync(link);

                    if (productPageElement == null)
                    {
                        continue;
                    }
                    IElement priceElement = productPageElement.QuerySelector("span.price");
                    string   priceString  = priceElement?.Attributes.Single(attr => attr.Name == "content")?.Value.Trim();
                    Decimal.TryParse(priceString, out productPrice);
                    IElement descriptionBlock = productPageElement?.QuerySelector("div.description_block_full");
                    productDescription = GetTextFromDomElement(descriptionBlock);
                    string imageLink = productPageElement?.QuerySelector("img.item_img")?.Attributes.Single(attr => attr.Name == "src")?.Value.Trim();
                    productImg = GetImgFromLink(startPageAddress + imageLink);
                }
                models.Add(new NewProductModel()
                {
                    Name        = productName,
                    Description = productDescription,
                    Price       = productPrice,
                    Image       = productImg
                });
            }
            return(models);
        }
コード例 #28
0
ファイル: Program.cs プロジェクト: Aleksandrov91/JokesFunApp
        private static void SandboxCode(IServiceProvider serviceProvider)
        {
            var dbContext = serviceProvider.GetService <JokesFunAppContext>();
            var config    = Configuration.Default.WithDefaultLoader();
            var context   = BrowsingContext.New(config);

            for (var i = 3001; i <= 10000; i++)
            {
                var url          = "http://fun.dir.bg/vic_open.php?id=" + i;
                var document     = context.OpenAsync(url).GetAwaiter().GetResult();
                var jokeContent  = document.QuerySelector("#newsbody")?.TextContent?.Trim();
                var categoryName = document.QuerySelector(".tag-links-left a")?.TextContent?.Trim();

                if (!string.IsNullOrWhiteSpace(jokeContent) && !string.IsNullOrWhiteSpace(categoryName))
                {
                    var category = dbContext.Categories.FirstOrDefault(x => x.Name == categoryName);

                    if (category == null)
                    {
                        category = new Category
                        {
                            Name = categoryName
                        };
                    }

                    var joke = new Joke
                    {
                        Category = category,
                        Content  = jokeContent
                    };

                    dbContext.Jokes.Add(joke);
                }

                if (i % 10 == 0)
                {
                    dbContext.SaveChanges();
                }

                Console.WriteLine($"{i} => {categoryName}");
            }
        }
コード例 #29
0
        private async void Button_Click(object sender, RoutedEventArgs e)
        {
            if (linkField != null)
            {
                var config = Configuration.Default.WithDefaultLoader();

                var document = await BrowsingContext.New(config).OpenAsync(linkField.Text);

                var rows     = document.QuerySelectorAll("tr.song");
                var songInfo = rows.Select(row => new SongInfo()
                {
                    Song   = row.QuerySelector("td:nth-child(2)")?.TextContent.Trim(),
                    Album  = row.QuerySelector("td:nth-child(4)")?.TextContent.Trim(),
                    Artist = row.QuerySelector("td:nth-child(3)")?.TextContent.Trim()
                });
                List <SongInfo> songs = songInfo.ToList();

                foreach (var song in songs)
                {
                    Console.WriteLine($"Artist-{song.Artist} Album-{song.Album} Song-{song.Song}");
                }
                string[] arr = new string[4];

                var gridView = new GridView();
                songView.View = gridView;
                gridView.Columns.Add(new GridViewColumn {
                    Header = "Artist", DisplayMemberBinding = new Binding("Artist")
                });
                gridView.Columns.Add(new GridViewColumn {
                    Header = "Song", DisplayMemberBinding = new Binding("Song")
                });
                gridView.Columns.Add(new GridViewColumn {
                    Header = "Album", DisplayMemberBinding = new Binding("Album")
                });
                foreach (var song in songs)
                {
                    songView.Items.Add(new SongInfo {
                        Artist = song.Artist, Song = song.Song, Album = song.Album
                    });
                }
            }
        }
コード例 #30
0
        public void JavascriptErrorInListenerShouldNotThrowJavascriptException()
        {
            var config = Configuration.Default
                         .WithJs()
                         .WithOnly(ctx => new CustomEventLoop());

            var context = BrowsingContext.New(config);

            const string content = @"
                <html>
                    <body>
                        <script type='text/javascript'>
                            document.addEventListener('load', function() {
                                undefinedVariable.invalidMethod();
                            });
                        </script>
                        
                    </body>
                </html>
            ";

            Assert.DoesNotThrowAsync(async() =>
            {
                try
                {
                    await context.OpenAsync(r => r.Content(content));

                    var loop = (CustomEventLoop)context.GetService <IEventLoop>();
                    await Task.WhenAll(loop.Tasks);
                }
                catch (TargetInvocationException ex)
                {
                    // unmask exception
                    var innerAggregate = ex.InnerException as AggregateException;
                    if (innerAggregate != null)
                    {
                        throw innerAggregate.InnerException;
                    }
                    throw;
                }
            });
        }