예제 #1
0
        public async static void Run(string slidesQuery, string size, Action <string> action, CancellationToken token)
        {
            if (string.IsNullOrWhiteSpace(slidesQuery))
            {
                return;
            }

            var inputQueries = new AsyncQueue <string>();

            slidesQuery.Split('\n').ToList().ForEach(query => inputQueries.Enqueue(query));
            inputQueries.SetFinished();
            var outputURLs = new AsyncQueue <List <string> >();

            AsyncHelper.RunTasks(inputQueries, query => GetSlideURLs(query, size, token), outputURLs, token);

            var urls = new List <string>();

            while (await outputURLs.HasItemsAsync(token))
            {
                foreach (var item in outputURLs.Dequeue())
                {
                    urls.Add(item);
                }
            }

            var random = new Random();

            urls = urls.OrderBy(x => random.Next()).ToList();

            var inputURLs = new AsyncQueue <string>();

            urls.ForEach(url => inputURLs.Enqueue(url));
            inputURLs.SetFinished();

            SlideDownloader.Run(inputURLs, action, token);
        }
예제 #2
0
        async public static void Run(string username, string password, Action <string> action, CancellationToken token)
        {
            var urls = new AsyncQueue <string>();

            SlideDownloader.Run(urls, action, token);
            using (var browser = new WebBrowser())
                try
                {
                    var uri = new Uri("https://www.tumblr.com");

                    ClearCookies(uri);

                    browser.Navigate(uri);
                    await WaitForPageLoadAsync(browser, token);

                    await LoginToTumblr(username, password, browser, token);

                    var doc   = browser.Document;
                    var found = new HashSet <string>();
                    while (!token.IsCancellationRequested)
                    {
                        var posts = doc.GetElementById("posts");
                        if (posts == null)
                        {
                            break;
                        }

                        foreach (HtmlElement child in posts.Children)
                        {
                            if (child.GetAttribute("classname") != "post_container")
                            {
                                continue;
                            }

                            var ad = false;
                            foreach (HtmlElement divs in child.GetElementsByTagName("div"))
                            {
                                var className = $" {divs.GetAttribute("classname").ToLowerInvariant()} ";
                                if (className.Contains(" sponsored_post "))
                                {
                                    ad = true;
                                    break;
                                }
                            }
                            if (ad)
                            {
                                continue;
                            }

                            foreach (HtmlElement image in child.GetElementsByTagName("img"))
                            {
                                var className = image.GetAttribute("classname");
                                if ((className == "post_avatar_image") || (className == "reblog-avatar-image-thumb"))
                                {
                                    continue;
                                }

                                var src = new Uri(browser.Url, image.GetAttribute("src")).AbsoluteUri;
                                if (!found.Contains(src))
                                {
                                    urls.Enqueue(src);
                                }
                                found.Add(src);
                            }
                        }

                        if (found.Count >= 200)
                        {
                            urls.SetFinished();
                            break;
                        }

                        doc.Body.ScrollIntoView(false);

                        await Task.Delay(250);
                    }
                }
                catch { }
        }