private void SetSafeSearchSetting(GoogleImageSearchSettings.GoogleSafeSearchOptions gsso) { using (HttpUtility.CookieAwareWebClient client = new HttpUtility.CookieAwareWebClient(_cookies)) { //First we need to access the preferences page so we can get the special ID var response = client.DownloadString("http://images.google.com/preferences?hl=en"); //parse out signature var specialID = Regex.Match(response, "<input type=\"hidden\" name=\"sig\" value=\"(?<sig>.*?)\">").Groups["sig"].Value; //options are "on", "images", "off" var safeUIOption = ""; switch (gsso) { case GoogleImageSearchSettings.GoogleSafeSearchOptions.Moderate: safeUIOption = "images"; break; case GoogleImageSearchSettings.GoogleSafeSearchOptions.Off: safeUIOption = "off"; break; case GoogleImageSearchSettings.GoogleSafeSearchOptions.Strict: safeUIOption = "on"; break; } //set prefs string url = string.Format("http://images.google.com/setprefs?sig={0}&hl=en&lr=lang_en&uulo=1&muul=4_20&luul=&safeui={1}&suggon=1&newwindow=0&q=", specialID.Replace("=", "%3D"), safeUIOption); var finalResponse = client.DownloadString(url); } }
private void Authenticate(string username, string password) { //if we have a username/password and we aren't already authenticated then authenticate if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) { using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies)) { //check if the user is already logged in (doh!) try { var loginReg = @"<span class=""name"".*?" + username + "</span>"; string homepage = _client.DownloadString("http://wallbase.cc"); if (Regex.Match(homepage, loginReg, RegexOptions.IgnoreCase).Success) { return; } } catch (Exception ex) { Log.Logger.Write(string.Format("There was an error trying to check for a pre-existing wallbase auth, ignoring it. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors); } try { //need to extract the cross-site request forgery token from the page //<img.*src=""(?<img>.*(wallpaper.*\.(jpg|png)))"" var csrfRegex = new Regex(@"<input type=""hidden"" name=""csrf"" value=""(?<csrf>.*)"">"); var refWallbase64Regex = new Regex(@"<input type=""hidden"" name=""ref"" value=""(?<ref>.*)"">"); string loginPage = _client.DownloadString("http://wallbase.cc/user/login"); Match lpM = csrfRegex.Match(loginPage); Match lpWallbaseInbase64 = refWallbase64Regex.Match(loginPage); if (!lpM.Success) { return; } var loginData = new NameValueCollection(); loginData.Add("csrf", lpM.Groups["csrf"].Value); loginData.Add("ref", lpWallbaseInbase64.Groups["ref"].Value); loginData.Add("username", username); loginData.Add("password", password); _client.Referrer = "http://wallbase.cc/user/login"; _client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded"); byte[] result = _client.UploadValues(@"http://wallbase.cc/user/do_login", "POST", loginData); //we do not need the response, all we need are the cookies string response = System.Text.Encoding.UTF8.GetString(result); } catch (Exception ex) { throw new WallbaseAccessDeniedException("Wallbase authentication failed. Please verify your username and password.", ex); } } } }
private void Authenticate(string username, string password) { //if we have a username/password and we aren't already authenticated then authenticate if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) { using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies)) { string homepage = _client.DownloadString("http://alpha.wallhaven.cc/"); var dom = CQ.CreateDocument(homepage); //check if the user is already logged in (doh!) try { var loginReg = dom["span.username"]; if (loginReg.Length > 0) { return; } } catch (Exception ex) { Log.Logger.Write(string.Format("There was an error trying to check for a pre-existing wallhaven auth, ignoring it. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors); } try { var token = dom["input[name='_token']"].Val <string>(); var loginData = new NameValueCollection(); loginData.Add("_token", token); loginData.Add("username", username); loginData.Add("password", password); _client.Referrer = "http://alpha.wallhaven.cc/"; _client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded"); byte[] result = _client.UploadValues(@"http://alpha.wallhaven.cc/auth/login", "POST", loginData); //we do not need the response, all we need are the cookies string response = System.Text.Encoding.UTF8.GetString(result); } catch (Exception ex) { throw new WallbaseAccessDeniedException("Wallhaven authentication failed. Please verify your username and password.", ex); } } } }
private string GetDirectPictureUrl(string pageUrl) { using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies)) { var content = cawc.DownloadString(pageUrl); if (string.IsNullOrEmpty(content)) { return(string.Empty); } var dom = CQ.CreateDocument(content); var wall = dom["#wallpaper"]; if (wall.Length > 0) { return("http:" + wall[0].GetAttribute("src")); } return(string.Empty); } }
private void Authenticate(string username, string password) { //if we have a username/password and we aren't already authenticated then authenticate if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) { using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies)) { var loginData = new NameValueCollection(); loginData.Add("usrname", username); loginData.Add("pass", password); loginData.Add("nopass_email", "Type in your e-mail and press enter"); loginData.Add("nopass", "0"); // Hack: Authenticate the user twice! _client.UploadValues(@"http://wallbase.cc/user/login", "POST", loginData); var result = _client.UploadValues(@"http://wallbase.cc/user/login", "POST", loginData); //we do not need the response, all we need are the cookies //string response = System.Text.Encoding.UTF8.GetString(result); } } }
private string GetDirectPictureUrl(string pageUrl) { using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies)) { var content = cawc.DownloadString(pageUrl); if (string.IsNullOrEmpty(content)) { return(string.Empty); } //var regex = new Regex(@"<img.*src=""(?<img>.*(wallpaper.*\.(jpg|png)))"""); var regex = new Regex(@"\+B\('(?<img>.*?)'\)"); var m = regex.Match(content); if (m.Groups["img"].Success && !string.IsNullOrEmpty(m.Groups["img"].Value)) { byte[] decoded = Convert.FromBase64String(m.Groups["img"].Value); string final = Encoding.Default.GetString(decoded); return(final); } return(string.Empty); } }
private string GetDirectPictureUrl(string pageUrl) { using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies)) { var content = cawc.DownloadString(pageUrl); if (string.IsNullOrEmpty(content)) { return(string.Empty); } //<img id="wallpaper" src="http://alpha.wallhaven.cc/wallpapers/full/wallhaven-4010.jpg" alt="" draggable="false" data-wallpaper-id="4010" data-wallpaper-width="1920" data-wallpaper-height="1200" class="fill-horizontal" style="margin-top: 0px; margin-bottom: 0px;"> var regex = new Regex(@"<img.*src=""(?<img>.*(wallpapers.*\.(jpg|png)))""", RegexOptions.Singleline); //var regex = new Regex(@"\+B\('(?<img>.*?)'\)"); var m = regex.Match(content); if (m.Groups["img"].Success && !string.IsNullOrEmpty(m.Groups["img"].Value)) { return(m.Groups["img"].Value); //byte[] decoded = Convert.FromBase64String(m.Groups["img"].Value); //string final = Encoding.Default.GetString(decoded); //return final; } return(string.Empty); } }
public PictureList GetPictures(PictureSearch ps) { WallbaseImageSearchSettings wiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new WallbaseImageSearchSettings() : WallbaseImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig); //if max picture count is 0, then no maximum, else specified max var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue; int pageSize = wiss.GetPageSize(); int pageIndex = ps.PageToRetrieve; //set page to retreive if one is specified var imgFoundCount = 0; //authenticate to wallbase Authenticate(wiss.Username, wiss.Password); var wallResults = new List <Picture>(); string areaURL = wiss.BuildURL(); //string postParams = wiss.GetPostParams(search); do { //calculate page index. Random does not use pages, so for random just refresh with same url string strPageNum = (pageIndex > 0 && wiss.SA != "random") || (wiss.SA == "toplist" || wiss.SA == "user/collection" || wiss.SA == "user/favorites") ? (pageIndex * pageSize).ToString() : ""; string pageURL = areaURL.Contains("{0}") ? string.Format(areaURL, strPageNum) : areaURL; //string content = HttpPost(pageURL, postParams); string content = string.Empty; using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies)) { try { //if random then don't post values if (wiss.SA == "random") { content = _client.DownloadString(pageURL); } else { byte[] reqResult = _client.UploadValues(pageURL, wiss.GetPostParams()); content = System.Text.Encoding.Default.GetString(reqResult); } } catch (Exception ex) { Log.Logger.Write(string.Format("Failed to download search results from wallbase.cc, error: {0}", ex.ToString()), Log.LoggerLevels.Warnings); } } if (string.IsNullOrEmpty(content)) { break; } //parse html and get count var pics = ParsePictures(content); imgFoundCount = pics.Count(); //if we have an image ban list check for them // doing this in the provider instead of picture manager // ensures that our count does not go down if we have a max if (ps.BannedURLs != null && ps.BannedURLs.Count > 0) { pics = (from c in pics where !(ps.BannedURLs.Contains(c.Url)) select c).ToList(); } wallResults.AddRange(pics); //increment page index so we can get the next set of images if they exist pageIndex++; } while (imgFoundCount > 0 && wallResults.Count < maxPictureCount && ps.PageToRetrieve == 0); PictureList result = FetchPictures(wallResults, ps.PreviewOnly); result.Pictures = result.Pictures.Take(maxPictureCount).ToList(); return(result); }
public PictureList GetPictures(PictureSearch ps) { var result = new PictureList() { FetchDate = DateTime.Now }; //load provider search settings GoogleImageSearchSettings giss = GoogleImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig); //to help not have so many null checks, if no search settings provided then use default ones if (giss == null) { giss = new GoogleImageSearchSettings(); } //if search is empty, return now since we can't search without it if (string.IsNullOrEmpty(giss.Query)) { return(result); } var pageIndex = ps.PageToRetrieve; //set page to retrieve if one specified var imgFoundCount = 0; //if max picture count is 0, then no maximum, else specified max var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue; //build tbs strring var tbs = "";//isz:ex,iszw:{1},iszh:{2} //handle sizeing if (giss.ImageHeight > 0 && giss.ImageWidth > 0) { tbs += string.Format("isz:ex,iszw:{0},iszh:{1},", giss.ImageWidth.ToString(), giss.ImageHeight.ToString()); } //handle colors if (!string.IsNullOrEmpty(giss.Color)) { tbs += GoogleImageSearchSettings.GoogleImageColors.GetColorSearchString((from c in GoogleImageSearchSettings.GoogleImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ","; } //if we have a filter string then add it and trim off trailing commas if (!string.IsNullOrEmpty(tbs)) { tbs = ("&tbs=" + tbs).Trim(new char[] { ',' }); } //do safe search setup (off/strict/moderate) this is part of the session and tracked via cookies SetSafeSearchSetting(giss.GoogleSafeSearchOption); do { //build URL from query, dimensions and page index var url = string.Format(baseURL, giss.Query, tbs, (pageIndex * 20).ToString()); var response = string.Empty; using (HttpUtility.CookieAwareWebClient client = new HttpUtility.CookieAwareWebClient(_cookies)) { response = client.DownloadString(url); } var images = imagesRegex2.Matches(response); //track number of images found for paging purposes imgFoundCount = images.Count; //convert images found into picture entries foreach (Match item in images) { var purl = item.Groups["imgurlgrp"].Value; var referrer = item.Groups["imgrefgrp"].Value; var thumbnail = item.Groups["thumbURL"].Value; //get id and trim if necessary (ran into a few cases of rediculously long filenames) var id = System.IO.Path.GetFileNameWithoutExtension(purl); if (id.Length > 50) { id = id.Substring(0, 50); } //because google images come from so many sites it's not uncommon to have duplicate file names. (we fix this) id = string.Format("{0}_{1}", id, purl.GetHashCode().ToString()); Picture p = new Picture() { Url = purl, Id = id }; p.Properties.Add(Picture.StandardProperties.Thumbnail, thumbnail); p.Properties.Add(Picture.StandardProperties.Referrer, referrer); result.Pictures.Add(p); } //if we have an image ban list check for them // doing this in the provider instead of picture manager // ensures that our count does not go down if we have a max if (ps.BannedURLs != null && ps.BannedURLs.Count > 0) { result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList(); } //increment page index so we can get the next 20 images if they exist pageIndex++; // Max Picture count is defined in search settings passed in, check for it here too } while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount && ps.PageToRetrieve == 0); result.Pictures = result.Pictures.Take(maxPictureCount).ToList(); return(result); }