Exemplo n.º 1
0
        public PictureList GetPictures(PictureSearch ps)
        {
            WallbaseImageSearchSettings wiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new WallbaseImageSearchSettings() : WallbaseImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);

            //if max picture count is 0, then no maximum, else specified max
            var maxPictureCount = wiss.GetMaxImageCount(ps.MaxPictureCount);
            int pageSize = wiss.GetPageSize();
            int pageIndex = ps.PageToRetrieve; //set page to retreive if one is specified
            var imgFoundCount = 0;

            //authenticate to wallbase
            Authenticate(wiss.Username, wiss.Password);

            var wallResults = new List<Picture>();

            string areaURL = wiss.BuildURL();

            do
            {
                //calculate page index.  Random does not use pages, so for random just refresh with same url
                string strPageNum = (pageIndex * pageSize).ToString();

                string pageURL = areaURL.Contains("{0}") ? string.Format(areaURL, strPageNum) : areaURL;
                //string content = HttpPost(pageURL, postParams);
                string content = string.Empty;

                using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
                {
                    try
                    {
                        //if random then don't post values
                        //if (wiss.SA == "random")
                        //{
                            content = _client.DownloadString(pageURL);
                        //}
                        //else
                        //{
                        //    byte[] reqResult = _client.UploadValues(pageURL, wiss.GetPostParams());
                        //    content = System.Text.Encoding.Default.GetString(reqResult);
                        //}
                    }
                    catch (Exception ex)
                    {
                        Log.Logger.Write(string.Format("Failed to download search results from wallbase.cc, error: {0}", ex.ToString()), Log.LoggerLevels.Warnings);
                    }
                }

                if (string.IsNullOrEmpty(content))
                    break;

                //parse html and get count
                var pics = ParsePictures(content);
                imgFoundCount = pics.Count();

                //if we have an image ban list check for them
                // doing this in the provider instead of picture manager
                // ensures that our count does not go down if we have a max
                if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
                {
                    pics = (from c in pics where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
                }

                wallResults.AddRange(pics);

                //increment page index so we can get the next set of images if they exist
                pageIndex++;
            } while (imgFoundCount > 0 && wallResults.Count < maxPictureCount && ps.PageToRetrieve == 0);

            PictureList result = FetchPictures(wallResults, ps.PreviewOnly);
            result.Pictures = result.Pictures.Take(maxPictureCount).ToList();

            return result;
        }
Exemplo n.º 2
0
        private string GetDirectPictureUrl(string pageUrl)
        {
            using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies))
            {

                var content = cawc.DownloadString(pageUrl);
                if (string.IsNullOrEmpty(content)) return string.Empty;

                var regex = new Regex(@"<img.*src=""(?<img>.*(wallpaper.*\.(jpg|png)))""");
                //var regex = new Regex(@"\+B\('(?<img>.*?)'\)");
                var m = regex.Match(content);
                if (m.Groups["img"].Success && !string.IsNullOrEmpty(m.Groups["img"].Value))
                {
                    return m.Groups["img"].Value;
                    //byte[] decoded = Convert.FromBase64String(m.Groups["img"].Value);
                    //string final = Encoding.Default.GetString(decoded);
                    //return final;
                }

                return string.Empty;
            }
        }
Exemplo n.º 3
0
        public PictureList GetPictures(PictureSearch ps)
        {
            var result = new PictureList() { FetchDate = DateTime.Now };

            //load provider search settings
            GoogleImageSearchSettings giss = GoogleImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig) ??
                                             new GoogleImageSearchSettings();

            //if search is empty, return now since we can't search without it
            if (string.IsNullOrEmpty(giss.Query)) return result;

            var pageIndex = ps.PageToRetrieve; //set page to retrieve if one specified
            var imgFoundCount = 0;

            //if max picture count is 0, then no maximum, else specified max
            var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue;

            //build tbs strring
            var tbs = "";//isz:ex,iszw:{1},iszh:{2}

            //handle sizeing
            if (giss.ImageHeight > 0 && giss.ImageWidth > 0)
            {
                tbs += string.Format("isz:ex,iszw:{0},iszh:{1},", giss.ImageWidth, giss.ImageHeight);
            }

            //handle colors
            if (!string.IsNullOrEmpty(giss.Color))
            {
                tbs += GoogleImageSearchSettings.GoogleImageColors.GetColorSearchString((from c in GoogleImageSearchSettings.GoogleImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ",";
            }

            //if we have a filter string then add it and trim off trailing commas
            if (!string.IsNullOrEmpty(tbs)) tbs = ("&tbs=" + tbs).Trim(new char[]{','});

            //do safe search setup (off/strict/moderate) this is part of the session and tracked via cookies
            //SetSafeSearchSetting(giss.GoogleSafeSearchOption);

            do
            {
                //build URL from query, dimensions and page index
                var url = string.Format(baseURL, giss.Query, tbs, (pageIndex * 20));

                var response = string.Empty;
                using (var client = new HttpUtility.CookieAwareWebClient(_cookies))
                {
                    response = client.DownloadString(url);
                }

                var images = _imagesRegex2.Matches(response);

                //track number of images found for paging purposes
                imgFoundCount = images.Count;

                //convert images found into picture entries
                foreach (Match item in images)
                {
                    var purl = item.Groups["imgurlgrp"].Value;
                    var referrer = item.Groups["imgrefgrp"].Value;
                    var thumbnail = item.Groups["thumbURL"].Value;
                    //get id and trim if necessary (ran into a few cases of rediculously long filenames)
                    var id = System.IO.Path.GetFileNameWithoutExtension(purl);
                    if (id.Length > 50) id = id.Substring(0, 50);
                    //because google images come from so many sites it's not uncommon to have duplicate file names. (we fix this)
                    id = string.Format("{0}_{1}", id, purl.GetHashCode());

                    var p = new Picture() { Url = purl, Id = id };
                    p.Properties.Add(Picture.StandardProperties.Thumbnail, thumbnail);
                    p.Properties.Add(Picture.StandardProperties.Referrer, referrer);

                    result.Pictures.Add(p);
                }

                //if we have an image ban list check for them
                // doing this in the provider instead of picture manager
                // ensures that our count does not go down if we have a max
                if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
                {
                    result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
                }

                //increment page index so we can get the next 20 images if they exist
                pageIndex++;
                // Max Picture count is defined in search settings passed in, check for it here too
            } while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount && ps.PageToRetrieve == 0);

            result.Pictures = result.Pictures.Take(maxPictureCount).ToList();

            return result;
        }
Exemplo n.º 4
0
        private void Authenticate(string username, string password)
        {
            //if we have a username/password and we aren't already authenticated then authenticate
            if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password))
            {
                using(HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
                {
                    //check if the user is already logged in (doh!)
                    try
                    {
                        var loginReg = @"<span class=""name"".*?" + username + "</span>";
                        string homepage = _client.DownloadString("http://wallbase.cc");
                        if (Regex.Match(homepage, loginReg, RegexOptions.IgnoreCase).Success)
                        {
                            return;
                        }
                    }
                    catch(Exception ex)
                    {
                        Log.Logger.Write(string.Format("There was an error trying to check for a pre-existing wallbase auth, ignoring it.  Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors);
                    }

                    try
                    {
                        //need to extract the cross-site request forgery token from the page
                        //<img.*src=""(?<img>.*(wallpaper.*\.(jpg|png)))""
                        var csrfRegex = new Regex(@"<input type=""hidden"" name=""csrf"" value=""(?<csrf>.*)"">");
                        var refWallbase64Regex = new Regex(@"<input type=""hidden"" name=""ref"" value=""(?<ref>.*)"">");

                        string loginPage = _client.DownloadString("http://wallbase.cc/user/login");
                        Match lpM = csrfRegex.Match(loginPage);
                        Match lpWallbaseInbase64 = refWallbase64Regex.Match(loginPage);

                        if (!lpM.Success) return;

                        var loginData = new NameValueCollection();
                        loginData.Add("csrf", lpM.Groups["csrf"].Value);
                        loginData.Add("ref", lpWallbaseInbase64.Groups["ref"].Value);

                        loginData.Add("username", username);
                        loginData.Add("password", password);

                        _client.Referrer = "http://wallbase.cc/user/login";
                        _client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded");

                        byte[] result = _client.UploadValues(@"http://wallbase.cc/user/do_login", "POST", loginData);

                        //we do not need the response, all we need are the cookies
                        string response = System.Text.Encoding.UTF8.GetString(result);
                    }
                    catch (Exception ex)
                    {
                        throw new WallbaseAccessDeniedException("Wallbase authentication failed. Please verify your username and password.", ex);
                    }
                }
            }
        }
Exemplo n.º 5
0
        private void SetSafeSearchSetting(GoogleImageSearchSettings.GoogleSafeSearchOptions gsso)
        {
            using (var client = new HttpUtility.CookieAwareWebClient(_cookies))
            {
                //First we need to access the preferences page so we can get the special ID
                var response = client.DownloadString("http://images.google.com/preferences?hl=en");
                //parse out signature
                var specialID = Regex.Match(response, "<input type=\"hidden\" name=\"sig\" value=\"(?<sig>.*?)\">").Groups["sig"].Value;

                //options are "on", "images", "off"
                var safeUIOption = "";
                switch(gsso) {
                    case GoogleImageSearchSettings.GoogleSafeSearchOptions.Off:
                        safeUIOption = "off";
                        break;
                    case GoogleImageSearchSettings.GoogleSafeSearchOptions.On:
                        safeUIOption = "on";
                        break;
                }
                //set prefs
                string url = string.Format("http://images.google.com/setprefs?sig={0}&hl=en&lr=lang_en&uulo=1&muul=4_20&luul=&safeui={1}&suggon=1&newwindow=0&q=",
                                specialID.Replace("=", "%3D"), safeUIOption);

                var finalResponse = client.DownloadString(url);
            }
        }