Пример #1
0
    public void SaveThumbnailToDisk(string url, string filename)
    {
        var options = new EdgeOptions();

        options.AddArgument("headless");//Comment if we want to see the window.

        string     path   = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
        EdgeDriver driver = null;

        try
        {
            driver = new EdgeDriver(path, options);
            driver.Manage().Window.Size = new System.Drawing.Size(2000, 4000);
            driver.Navigate().GoToUrl(url);
            Thread.Sleep(5000);
            var screenshot = (driver as ITakesScreenshot).GetScreenshot();

            Log.Information("Saving thumbnail file '{filename}'", filename);
            screenshot.SaveAsFile(filename);
        }
        catch (Exception ex)
        {
            Log.Error(ex, "ERROR: Unable to save webpage thumbnail");
        }
        finally
        {
            if (driver != null)
            {
                driver.Close();
                driver.Quit();
            }
        }
    }
Пример #2
0
        private RemoteWebDriver Initialize(bool useFirefox = false, bool isMobile = false)
        {
            RemoteWebDriver driver;

            if (useFirefox)
            {
                var firefoxOptions = new FirefoxOptions();
                firefoxOptions.SetPreference(Constants.PrivateBrowsingKey, true);
                if (isMobile)
                {
                    firefoxOptions.SetPreference(Constants.UserAgentKey, Constants.MobileUserAgent);
                }
                driver = new FirefoxDriver(firefoxOptions);
            }
            else
            {
                EdgeOptions driverOptions = new EdgeOptions {
                    UseChromium = true
                };

                if (isMobile)
                {
                    driverOptions.AddArgument($"{ Constants.EdgeUserAgentArgument}={Constants.MobileUserAgent}");
                }
                driver = new EdgeDriver(driverOptions);
                driver.Manage().Cookies.DeleteAllCookies();
            }

            driver.Manage().Window.Maximize();

            return(driver);
        }
Пример #3
0
        public override IWebDriver GetDriver()
        {
            var options = new EdgeOptions();

            options.UseChromium = true;
            options.AddArgument("ignore-certificate-errors");
            var service = EdgeDriverService.CreateChromiumService(DriversPath);

            return(new OpenQA.Selenium.Edge.EdgeDriver(service, options, TimeSpan.FromMinutes(3)));
        }
Пример #4
0
        public EdgeOptions ToEdgeOptions(EdgeBrowserSettings settings)
        {
            var result = new EdgeOptions();

            if (settings.Options.UseInPrivateBrowsing)
            {
                result.AddArgument("inprivate");
            }
            return(result);
        }
Пример #5
0
        public void CreateWebDriver(ScenarioContext context)
        {
            var currentDirectory = System.IO.Directory.GetCurrentDirectory();

            System.Diagnostics.Debug.WriteLine($"AdminAssistant.Blazor.Test - The Directory where your webdriver exe should be placed is: {currentDirectory}");

            // Launch Microsoft Edge (Chromium) - See https://github.com/microsoft/edge-selenium-tools
            // Requires the WebDriver for your version of Edge (chromium) - see https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/#downloads
            var options = new EdgeOptions();

            options.UseChromium = true;
            options.AddArgument("--start-maximized");
            options.AddArgument("--disable-notifications");

            IWebDriver webDriver = new EdgeDriver(options);

            webDriver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(5);
            context["WEB_DRIVER"] = webDriver;
        }
        public static EdgeOptions EdgeOptions()
        {
            var options = new EdgeOptions();

            options.AddArgument("start-maximized");
            options.UseChromium          = true;
            options.PageLoadStrategy     = PageLoadStrategy.Normal;
            options.UseInPrivateBrowsing = true;

            return(options);
        }
Пример #7
0
        private IWebDriver SetupEdgeDriver()
        {
            var ed = new EdgeOptions();

            ed.AddArgument("headless");
#if DEBUG
            new DriverManager().SetUpDriver(new EdgeConfig());
            return(new EdgeDriver(ed));
#else
            var envEdgeWebDriver = Environment.GetEnvironmentVariable("EdgeWebDriver");
            return(new EdgeDriver(envEdgeWebDriver, ed));
#endif
        }
        public static EdgeOptions GetEdgeOptions(string lang, bool headless = false, PlatformType platformType = PlatformType.Any)
        {
            EdgeOptions options = new EdgeOptions();

            options.UseChromium = true;
            options.AddArguments("disable-gpu", $"--lang={lang}");
            if (headless)
            {
                options.AddArgument("headless");
            }
            SetPlatform(options, platformType);
            return(options);
        }
Пример #9
0
        private static IWebDriver CreateDriver()
        {
            try
            {
                var programDirectory = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
                var edgeService      = EdgeDriverService.CreateChromiumService($"{programDirectory}\\Drivers", $"{DriverName}.exe");
                edgeService.SuppressInitialDiagnosticInformation = true;
                edgeService.HideCommandPromptWindow = true;

                var options = new EdgeOptions {
                    UseChromium = true
                };
                options.AddArgument("headless");
                options.AddArgument("disable-gpu");

                return(new EdgeDriver(edgeService, options));
            }
            catch (InvalidOperationException)
            {
                Console.WriteLine("Unable to download due to MicrosoftEdge not up to date. Please update it before proceeding.");
                throw;
            }
        }
Пример #10
0
        public void Test()
        {
            ChromeOptions chromeOptions = new ChromeOptions();

            chromeOptions.AddArgument("--headless");
            EdgeOptions edgeOptions = new EdgeOptions();

            edgeOptions.AddArgument("--headless");

            //driver = new ChromeDriver(chromeOptions);
            driver = new EdgeDriver(edgeOptions);

            googleMaps = new GoogleMapsPageObjects(driver);
            googleMaps.GoToPage();
            googleMaps.ClickModal();
            googleMaps.Route.Click();
        }
Пример #11
0
        public static RemoteWebDriver Create(BrowserType driverType, bool headless = false)
        {
            var             headlessArg = "--headless";
            RemoteWebDriver driver;

            switch (driverType)
            {
            case BrowserType.Chrome:
                var chromeOption = new ChromeOptions();
                if (headless)
                {
                    chromeOption.AddArgument(headlessArg);
                }
                driver = new ChromeDriver(chromeOption);

                break;

            case BrowserType.Firefox:
                var firefoxOption = new FirefoxOptions();
                if (headless)
                {
                    firefoxOption.AddArgument(headlessArg);
                }
                driver = new FirefoxDriver(firefoxOption);

                break;

            case BrowserType.Edge:
                var edgeOption = new EdgeOptions();
                if (headless)
                {
                    edgeOption.UseChromium = true;
                    edgeOption.AddArgument(headlessArg);
                }
                driver = new EdgeDriver(edgeOption);

                break;

            default:
                throw new ArgumentException($"Driver type not supported: {driverType}");
            }

            driver.Manage().Window.Maximize();

            return(driver);
        }
Пример #12
0
    public (string, Uri) WebDriverUrlToDisk(string url, string urlHash, string filename)
    {
        Log.Debug("Loading Selenium URL '{urlHash}':'{url}'", urlHash, url);

        var options = new EdgeOptions();

        options.AddArgument("headless");//Comment if we want to see the window.

        string     path   = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
        EdgeDriver driver = null;

        try
        {
            driver = new EdgeDriver(path, options);
            driver.Navigate().GoToUrl(url);

            // Delete the file if it already exists
            if (File.Exists(filename))
            {
                File.Delete(filename);
            }

            Log.Debug("Saving text file '{fileName}'", filename);
            File.WriteAllText(filename, driver.PageSource);

            return(filename, new Uri(driver.Url));
        }
        catch (Exception ex)
        {
            Log.Error(ex, "WebDriverUrlToDisk: Unexpected error '{message}'", ex.Message);
        }
        finally
        {
            if (driver != null)
            {
                driver.Close();
                driver.Quit();
            }
        }

        return(string.Empty, new Uri(url));
    }
Пример #13
0
        private static EdgeDriver EdgeDriver(Device device)
        {
            EdgeOptions options = new EdgeOptions
            {
                UseChromium = true
            };
            EdgeDriver _driver = null;

            switch (device)
            {
            case Device.Desktop:
                options.AddArgument("--start-maximized");
                _driver = new EdgeDriver(driversPath, options);
                break;

            case Device.Tablet:
                options.EnableMobileEmulation("iPad");
                _driver = new EdgeDriver(driversPath, options);
                break;
            }
            return(_driver);
        }
Пример #14
0
        private static EdgeDriver EdgeDriver(Device device)
        {
            EdgeDriverService service = EdgeDriverService.CreateDefaultService(driversPath, "msedgedriver.exe");
            EdgeOptions       options = new EdgeOptions();
            EdgeDriver        _driver = null;

            switch (device)
            {
            case Device.Desktop:
                _driver = new EdgeDriver(service);
                _driver.Manage().Window.Size = new Size(1200, 700);
                break;

            case Device.Tablet:
                options.UseChromium = true;
                options.EnableMobileEmulation("iPad");
                options.AddArgument("--window-size=768,700");
                _driver = new EdgeDriver(driversPath, options);
                break;
            }
            return(_driver);
        }
Пример #15
0
        /// <summary>
        /// Set the WebDriver browser by browser name.
        /// </summary>
        /// <param name="browserName">Options not case sensitive and are: "chrome", "firefox", "chromeheadless", "firefoxheadless", "ie"</param>
        /// <exception cref="ArgumentException"></exception>
        /// <returns>Returns a the appropriate WebDriver by browser name</returns>
        private IWebDriver SetBrowser(string browserName)
        {
            if (IsRemote)
            {
                switch (browserName.ToLower())
                {
                case "firefox":
                    var rfireoptions = AcceptUknownIssuer();
                    return(new RemoteWebDriver(RemoteHubUri, rfireoptions.ToCapabilities(), GlobalWaitTime));

                case "chrome":
                    var rcoptions = new ChromeOptions();
                    return(new RemoteWebDriver(RemoteHubUri, rcoptions.ToCapabilities(), GlobalWaitTime));


                case "edge":
                    var edgeHeadlessOptions = new EdgeOptions();
                    edgeHeadlessOptions.UseChromium = true;
                    return(new RemoteWebDriver(RemoteHubUri, edgeHeadlessOptions.ToCapabilities(), GlobalWaitTime));


                default:
                    throw new ArgumentException("Provide a valid browser name.");
                }
            }
            else
            {
                switch (browserName.ToLower())
                {
                case "firefox":
                    FirefoxDriverService fireservice = FirefoxDriverService.CreateDefaultService(BaseApplicationPath);
                    var fireoptions = AcceptUknownIssuer();
                    return(new FirefoxDriver(fireservice, fireoptions, GlobalWaitTime));

                case "chrome":
                    var coptions = new ChromeOptions();
                    return(new ChromeDriver(BaseApplicationPath, coptions, GlobalWaitTime));

                case "firefoxheadless":
                    FirefoxDriverService firehservice = FirefoxDriverService.CreateDefaultService(BaseApplicationPath);
                    var firehoptions = AcceptUknownIssuer();
                    firehoptions.AddArgument("--headless");
                    return(new FirefoxDriver(firehservice, firehoptions, GlobalWaitTime));

                case "chromeheadless":
                    var choptions = new ChromeOptions();
                    choptions.AddArgument("headless");
                    choptions.AddArgument("window-size=1200x800");
                    return(new ChromeDriver(BaseApplicationPath, choptions, GlobalWaitTime));

                case "edge":
                    var edgeOptions = new EdgeOptions();
                    edgeOptions.UseChromium = true;
                    return(new EdgeDriver(BaseApplicationPath, edgeOptions, GlobalWaitTime));

                case "edgeheadless":
                    var edgeHeadlessOptions = new EdgeOptions();
                    edgeHeadlessOptions.UseChromium = true;
                    edgeHeadlessOptions.AddArgument("--headless --disable-gpu");
                    return(new EdgeDriver(BaseApplicationPath, edgeHeadlessOptions, GlobalWaitTime));

                default:
                    throw new ArgumentException("Provide a valid browser name.");
                }
            }
        }
Пример #16
0
        public static List <string[]> GetRightStufAnimeData(string bookTitle, char bookType, bool memberStatus, byte currPageNum)
        {
            // Initialize the html doc for crawling
            HtmlDocument doc = new HtmlDocument();

            EdgeOptions edgeOptions = new EdgeOptions();

            edgeOptions.UseChromium      = true;
            edgeOptions.PageLoadStrategy = PageLoadStrategy.Eager;
            edgeOptions.AddArgument("headless");
            edgeOptions.AddArgument("disable-gpu");
            edgeOptions.AddArgument("disable-extensions");
            edgeOptions.AddArgument("inprivate");
            EdgeDriver edgeDriver = new EdgeDriver(edgeOptions);

            edgeDriver.Navigate().GoToUrl(getUrl(bookType, currPageNum, bookTitle));
            Thread.Sleep(3500);
            doc.LoadHtml(edgeDriver.PageSource);

            // Get the page data from the HTML doc
            HtmlNodeCollection titleData       = doc.DocumentNode.SelectNodes("//span[@itemprop='name']");
            HtmlNodeCollection priceData       = doc.DocumentNode.SelectNodes("//span[@itemprop='price']");
            HtmlNodeCollection stockStatusData = doc.DocumentNode.SelectNodes("//div[@class='product-line-stock-container '] | //span[@class='product-line-stock-msg-out-text']");
            HtmlNode           pageCheck       = doc.DocumentNode.SelectSingleNode("//li[@class='global-views-pagination-next']");

            try{
                double  GotAnimeDiscount = 0.05;
                decimal priceVal;
                string  priceTxt, stockStatus, currTitle;
                Regex   removeWords = new Regex(@"[^a-z']");
                for (int x = 0; x < titleData.Count; x++)
                {
                    currTitle = titleData[x].InnerText;
                    if (removeWords.Replace(currTitle.ToLower(), "").IndexOf(removeWords.Replace(bookTitle.ToLower(), "")) == 0)
                    {
                        priceVal = System.Convert.ToDecimal(priceData[x].InnerText.Substring(1));
                        priceTxt = memberStatus ? "$" + (priceVal - (priceVal * (decimal)GotAnimeDiscount)).ToString("0.00") : priceData[x].InnerText;

                        stockStatus = stockStatusData[x].InnerText;
                        if (stockStatus.IndexOf("In Stock") != -1)
                        {
                            stockStatus = "IS";
                        }
                        else if (stockStatus.IndexOf("Out of Stock") != -1)
                        {
                            stockStatus = "OOS";
                        }
                        else if (stockStatus.IndexOf("Pre-Order") != -1)
                        {
                            stockStatus = "PO";
                        }
                        else
                        {
                            stockStatus = "OOP";
                        }

                        dataList.Add(new string[] { currTitle, priceTxt.Trim(), stockStatus, "RightStufAnime" });
                        //Console.WriteLine(currTitle + " " + priceTxt + " " + stockStatus);
                    }
                }

                if (pageCheck != null)
                {
                    currPageNum++;
                    GetRightStufAnimeData(bookTitle, bookType, memberStatus, currPageNum);
                }
                else
                {
                    edgeDriver.Quit();

                    foreach (string link in links)
                    {
                        Console.WriteLine(link);
                    }
                }
            }
            catch (NullReferenceException ex) {
                Console.Error.WriteLine(ex);
                Environment.Exit(1);
            }

            using (StreamWriter outputFile = new StreamWriter(@"C:\MangaWebScrape\MangaWebScrape\Data_Files\RightStufAnimeData.txt"))
            {
                foreach (string[] data in dataList)
                {
                    outputFile.WriteLine(data[0] + " " + data[1] + " " + data[2] + " " + data[3]);
                }
            }

            return(dataList);
        }
Пример #17
0
        public static List <string[]> GetBarnesAndNobleData(string bookTitle, char bookType, byte currPageNum)
        {
            // Initialize the html doc for crawling
            HtmlDocument doc = new HtmlDocument();

            EdgeOptions edgeOptions = new EdgeOptions();

            edgeOptions.UseChromium      = true;
            edgeOptions.PageLoadStrategy = PageLoadStrategy.Eager;
            edgeOptions.AddArgument("headless");
            edgeOptions.AddArgument("disable-gpu");
            edgeOptions.AddArgument("disable-extensions");
            edgeOptions.AddArgument("inprivate");
            EdgeDriver edgeDriver = new EdgeDriver(edgeOptions);

            edgeDriver.Navigate().GoToUrl(GetUrl(bookTitle, currPageNum, bookType));
            Thread.Sleep(2000);
            doc.LoadHtml(edgeDriver.PageSource);

            HtmlNodeCollection titleData = doc.DocumentNode.SelectNodes("//a[@class=' ']");

            Console.WriteLine();
            HtmlNodeCollection priceData       = doc.DocumentNode.SelectNodes("//a[@class=' link']//span[last()]");
            HtmlNodeCollection stockStatusData = doc.DocumentNode.SelectNodes("//div[1][@class='availability-spacing flex']//p");
            HtmlNode           pageCheck       = doc.DocumentNode.SelectSingleNode("//li[@class='pagination__next ']");

            if (bookType == 'N')
            {
                HtmlNodeCollection formatTypeData = doc.DocumentNode.SelectNodes("//span[@class='format']");
                for (int x = 0; x < formatTypeData.Count; x++)
                {
                    if (formatTypeData[x].InnerText.IndexOf("NOOK") != -1)
                    {
                        titleData.RemoveAt(x);
                        formatTypeData.RemoveAt(x);
                        x--;
                    }
                }
                formatTypeData = null; //Free the format type list from memory
            }
            try{
                string stockStatus, currTitle;
                Regex  removeExtra = new Regex(@"[^a-z']");
                for (int x = 0; x < titleData.Count; x++)
                {
                    currTitle = titleData[x].InnerText;
                    if (removeExtra.Replace(currTitle.ToLower(), "").IndexOf(removeExtra.Replace(bookTitle.ToLower(), "")) == 0)
                    {
                        stockStatus = stockStatusData[x].InnerText;
                        if (stockStatus.IndexOf("Available Online") != -1)
                        {
                            stockStatus = "IS";
                        }
                        else if (stockStatus.IndexOf("Out of Stock Online") != -1)
                        {
                            stockStatus = "OOS";
                        }
                        else if (stockStatus.IndexOf("Pre-order Now") != -1)
                        {
                            stockStatus = "PO";
                        }

                        dataList.Add(new string[] { currTitle, priceData[x].InnerText.Trim(), stockStatus, "Barnes & Noble" });
                    }
                }

                if (pageCheck != null)
                {
                    currPageNum++;
                    GetBarnesAndNobleData(bookTitle, bookType, currPageNum);
                }
                else
                {
                    edgeDriver.Quit();

                    foreach (string link in links)
                    {
                        Console.WriteLine(link);
                    }
                }
            }
            catch (NullReferenceException ex) {
                Console.Error.WriteLine(ex);
                Environment.Exit(1);
            }

            using (StreamWriter outputFile = new StreamWriter(@"C:\MangaWebScrape\MangaWebScrape\Data_Files\BarnesAndNobleData.txt"))
            {
                foreach (string[] data in dataList)
                {
                    outputFile.WriteLine(data[0] + " " + data[1] + " " + data[2] + " " + data[3]);
                }
            }

            return(dataList);
        }
Пример #18
0
        private async Task BingSearchAsync(RewardType rewardType, int current, int target, bool useFirefox = false)
        {
            try
            {
                wordList = await DownloadJsonDataAsync <List <string> >(Constants.WordsListUrl);

                //Edge browser
                if (rewardType == RewardType.EdgeBonus || !useFirefox)
                {
                    var options = new EdgeOptions
                    {
                        UseChromium = true,
                    };

                    if (rewardType == RewardType.Mobile)
                    {
                        options.AddArgument($"{ Constants.EdgeUserAgentArgument}={Constants.MobileUserAgent}");
                    }

                    using (var edgeDriver = new EdgeDriver(options))
                    {
                        var edgeWait = new WebDriverWait(edgeDriver, TimeSpan.FromSeconds(30));

                        await LoginAsync(edgeDriver, edgeWait);

                        Search(edgeDriver, edgeWait, Constants.BingSearchURL + "Give me Edge points");

                        await Task.Delay(2000);

                        try
                        {
                            if (rewardType == RewardType.Mobile)
                            {
                                var hamburg = edgeWait.Until(d => d.FindElement(By.Id(Constants.MHamburger)));
                                hamburg?.Click();

                                var signin = edgeWait.Until(d => d.FindElement(By.Id(Constants.HbS)));
                                signin?.Click();

                                await Task.Delay(1500);
                            }
                            else
                            {
                                try
                                {
                                    var id_p = edgeWait.Until(d => d.FindElement(By.Id(Constants.ID_P)));
                                    if (id_p != null)
                                    {
                                        id_p?.Click();
                                    }
                                }
                                catch (Exception ex)
                                {
                                    Debug.WriteLine(ex.Message);
                                    Debug.WriteLine(ex.StackTrace);
                                    var id_a = edgeWait.Until(d => d.FindElement(By.Id(Constants.ID_A)));
                                    if (id_a != null)
                                    {
                                        id_a?.Click();
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Debug.WriteLine(ex.Message);
                            Debug.WriteLine(ex.StackTrace);
                        }

                        while (current < target)
                        {
                            var nextInt = rand.Next(wordList.Count + 1);
                            Search(edgeDriver, edgeWait, Constants.BingSearchURL + wordList[nextInt <= wordList.Count ? nextInt : 0]);
                            current += 5;
                            if (current >= target)
                            {
                                var currentBreakDown = CheckBreakDown(edgeDriver, edgeWait);

                                if (currentBreakDown.ContainsKey(rewardType))
                                {
                                    current = currentBreakDown[rewardType].Current;
                                    Console.WriteLine("{0} points of {1} completed", currentBreakDown[rewardType].Current, currentBreakDown[rewardType].Maximum);
                                }
                            }
                        }
                        edgeDriver?.Quit();
                    }
                }
                //Use Firefox
                else
                {
                    var      options = new FirefoxOptions();
                    TimeSpan timeout = TimeSpan.FromSeconds(60);
                    if (rewardType == RewardType.Mobile)
                    {
                        options.SetPreference(Constants.UserAgentKey, Constants.MobileUserAgent);
                    }

                    options.SetPreference(Constants.PrivateBrowsingKey, true);
                    using (var firefoxDriver = new FirefoxDriver(options))
                    {
                        WebDriverWait driverWait = new WebDriverWait(firefoxDriver, timeout);
                        await LoginAsync(firefoxDriver, driverWait);

                        firefoxDriver.Navigate().GoToUrl(Constants.BingSearchURL + "Edge Points");

                        await Task.Delay(2000);

                        try
                        {
                            if (rewardType == RewardType.Mobile)
                            {
                                var hamburg = driverWait.Until(d => d.FindElement(By.Id(Constants.MHamburger)));
                                hamburg?.Click();

                                var signin = driverWait.Until(d => d.FindElement(By.Id(Constants.HbS)));
                                signin?.Click();

                                await Task.Delay(1500);
                            }
                            else
                            {
                                try
                                {
                                    var id_p = driverWait.Until(d => d.FindElement(By.Id(Constants.ID_P)));
                                    if (id_p != null)
                                    {
                                        id_p.Click();
                                    }
                                }
                                catch (Exception ex)
                                {
                                    Debug.WriteLine(ex.Message);
                                    Debug.WriteLine(ex.StackTrace);
                                    var id_a = driverWait.Until(d => d.FindElement(By.Id(Constants.ID_A)));
                                    if (id_a != null)
                                    {
                                        id_a.Click();
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Debug.WriteLine(ex.Message);
                            Debug.WriteLine(ex.StackTrace);
                        }

                        while (current < target)
                        {
                            var nextInt = rand.Next(wordList.Count + 1);
                            Search(firefoxDriver, driverWait, Constants.BingSearchURL + wordList[nextInt <= wordList.Count ? nextInt : 0]);
                            current += 5;
                            if (current >= target)
                            {
                                var currentBreakDown = CheckBreakDown(firefoxDriver, driverWait);

                                if (currentBreakDown.ContainsKey(rewardType))
                                {
                                    current = currentBreakDown[rewardType].Current;
                                    Console.WriteLine($"{currentBreakDown[rewardType].Current} points of {currentBreakDown[rewardType].Maximum} completed");
                                }
                            }
                        }
                        firefoxDriver?.Quit();
                    }
                }
            }
            catch (Exception ex)
            {
                Debug.WriteLine(ex.Message);
                Debug.WriteLine(ex.StackTrace);
            }
        }
Пример #19
0
        public static List <string[]> GetRobertsAnimeCornerStoreData(string bookTitle, char bookType)
        {
            EdgeOptions edgeOptions = new EdgeOptions();

            edgeOptions.UseChromium      = true;
            edgeOptions.PageLoadStrategy = PageLoadStrategy.Eager;
            edgeOptions.AddArgument("headless");
            edgeOptions.AddArgument("disable-gpu");
            edgeOptions.AddArgument("disable-extensions");
            edgeOptions.AddArgument("inprivate");
            EdgeDriver edgeDriver = new EdgeDriver(edgeOptions);

            // Initialize the html doc for crawling
            HtmlDocument doc = new HtmlDocument();

            string linkPage = getPageData(edgeDriver, bookTitle, bookType, doc);

            if (linkPage == null)
            {
                Console.Error.WriteLine("Error! Invalid Series Title");
                Environment.Exit(1);
            }
            else if (linkPage.Equals("DNE"))
            {
                Console.Error.WriteLine(bookTitle + " does not exist at this website");
                edgeDriver.Quit();
            }
            else
            {
                try{
                    // Start scraping the URL where the data is found
                    edgeDriver.Navigate().GoToUrl(linkPage);
                    Thread.Sleep(2000);

                    // Get the html doc for crawling
                    doc.LoadHtml(edgeDriver.PageSource);

                    List <HtmlNode> titleData = doc.DocumentNode.SelectNodes("//font[@face='dom bold, arial, helvetica']//b").Where(title => title.InnerText.ToLower().IndexOf(bookTitle.ToLower()) != -1).ToList();
                    List <HtmlNode> priceData = doc.DocumentNode.SelectNodes("//font[@color='#ffcc33']").Where(price => price.InnerText.IndexOf("$") != -1).ToList();

                    string currTitle;
                    Regex  pattern = new Regex(@"#[\d]+( )");
                    for (int x = 0; x < titleData.Count; x++)
                    {
                        currTitle = titleData[x].InnerText.Replace(",", "");
                        currTitle = currTitle.Substring(0, pattern.Match(currTitle).Groups[1].Index);

                        dataList.Add(new string[] { currTitle, priceData[x].InnerText.Trim(), currTitle.IndexOf("Pre Order") != -1 ? "PO" : "IS", "RobertsAnimeCornerStore" });
                    }

                    foreach (string link in links)
                    {
                        Console.WriteLine(link);
                    }

                    edgeDriver.Quit();
                }
                catch (NullReferenceException ex) {
                    Console.Error.WriteLine(ex);
                    Environment.Exit(1);
                }
            }

            using (StreamWriter outputFile = new StreamWriter(@"C:\MangaWebScrape\MangaWebScrape\Data_Files\RobertsAnimeCornerStoreData.txt"))
            {
                foreach (string[] data in dataList)
                {
                    outputFile.WriteLine(data[0] + " " + data[1] + " " + data[2] + " " + data[3]);
                }
            }

            return(dataList);
        }