private void SetSafeSearchSetting(GoogleImageSearchSettings.GoogleSafeSearchOptions gsso)
{
using (HttpUtility.CookieAwareWebClient client = new HttpUtility.CookieAwareWebClient(_cookies))
{
//First we need to access the preferences page so we can get the special ID
var response = client.DownloadString("http://images.google.com/preferences?hl=en");
//parse out signature
var specialID = Regex.Match(response, "<input type=\"hidden\" name=\"sig\" value=\"(?<sig>.*?)\">").Groups["sig"].Value;
//options are "on", "images", "off"
var safeUIOption = "";
switch (gsso)
{
case GoogleImageSearchSettings.GoogleSafeSearchOptions.Moderate:
safeUIOption = "images";
break;
case GoogleImageSearchSettings.GoogleSafeSearchOptions.Off:
safeUIOption = "off";
break;
case GoogleImageSearchSettings.GoogleSafeSearchOptions.Strict:
safeUIOption = "on";
break;
}
//set prefs
string url = string.Format("http://images.google.com/setprefs?sig={0}&hl=en&lr=lang_en&uulo=1&muul=4_20&luul=&safeui={1}&suggon=1&newwindow=0&q=",
specialID.Replace("=", "%3D"), safeUIOption);
var finalResponse = client.DownloadString(url);
}
}
private void Authenticate(string username, string password)
{
//if we have a username/password and we aren't already authenticated then authenticate
if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password))
{
using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
{
//check if the user is already logged in (doh!)
try
{
var loginReg = @"<span class=""name"".*?" + username + "</span>";
string homepage = _client.DownloadString("http://wallbase.cc");
if (Regex.Match(homepage, loginReg, RegexOptions.IgnoreCase).Success)
{
return;
}
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("There was an error trying to check for a pre-existing wallbase auth, ignoring it. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors);
}
try
{
//need to extract the cross-site request forgery token from the page
//<img.*src=""(?<img>.*(wallpaper.*\.(jpg|png)))""
var csrfRegex = new Regex(@"<input type=""hidden"" name=""csrf"" value=""(?<csrf>.*)"">");
var refWallbase64Regex = new Regex(@"<input type=""hidden"" name=""ref"" value=""(?<ref>.*)"">");
string loginPage = _client.DownloadString("http://wallbase.cc/user/login");
Match lpM = csrfRegex.Match(loginPage);
Match lpWallbaseInbase64 = refWallbase64Regex.Match(loginPage);
if (!lpM.Success)
{
return;
}
var loginData = new NameValueCollection();
loginData.Add("csrf", lpM.Groups["csrf"].Value);
loginData.Add("ref", lpWallbaseInbase64.Groups["ref"].Value);
loginData.Add("username", username);
loginData.Add("password", password);
_client.Referrer = "http://wallbase.cc/user/login";
_client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded");
byte[] result = _client.UploadValues(@"http://wallbase.cc/user/do_login", "POST", loginData);
//we do not need the response, all we need are the cookies
string response = System.Text.Encoding.UTF8.GetString(result);
}
catch (Exception ex)
{
throw new WallbaseAccessDeniedException("Wallbase authentication failed. Please verify your username and password.", ex);
}
}
}
}
private void Authenticate(string username, string password)
{
//if we have a username/password and we aren't already authenticated then authenticate
if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password))
{
using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
{
string homepage = _client.DownloadString("http://alpha.wallhaven.cc/");
var dom = CQ.CreateDocument(homepage);
//check if the user is already logged in (doh!)
try
{
var loginReg = dom["span.username"];
if (loginReg.Length > 0)
{
return;
}
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("There was an error trying to check for a pre-existing wallhaven auth, ignoring it. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors);
}
try
{
var token = dom["input[name='_token']"].Val <string>();
var loginData = new NameValueCollection();
loginData.Add("_token", token);
loginData.Add("username", username);
loginData.Add("password", password);
_client.Referrer = "http://alpha.wallhaven.cc/";
_client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded");
byte[] result = _client.UploadValues(@"http://alpha.wallhaven.cc/auth/login", "POST", loginData);
//we do not need the response, all we need are the cookies
string response = System.Text.Encoding.UTF8.GetString(result);
}
catch (Exception ex)
{
throw new WallbaseAccessDeniedException("Wallhaven authentication failed. Please verify your username and password.", ex);
}
}
}
}
private string GetDirectPictureUrl(string pageUrl)
{
using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies))
{
var content = cawc.DownloadString(pageUrl);
if (string.IsNullOrEmpty(content))
{
return(string.Empty);
}
var dom = CQ.CreateDocument(content);
var wall = dom["#wallpaper"];
if (wall.Length > 0)
{
return("http:" + wall[0].GetAttribute("src"));
}
return(string.Empty);
}
}
private void Authenticate(string username, string password)
{
//if we have a username/password and we aren't already authenticated then authenticate
if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password))
{
using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
{
var loginData = new NameValueCollection();
loginData.Add("usrname", username);
loginData.Add("pass", password);
loginData.Add("nopass_email", "Type in your e-mail and press enter");
loginData.Add("nopass", "0");
// Hack: Authenticate the user twice!
_client.UploadValues(@"http://wallbase.cc/user/login", "POST", loginData);
var result = _client.UploadValues(@"http://wallbase.cc/user/login", "POST", loginData);
//we do not need the response, all we need are the cookies
//string response = System.Text.Encoding.UTF8.GetString(result);
}
}
}
private string GetDirectPictureUrl(string pageUrl)
{
using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies))
{
var content = cawc.DownloadString(pageUrl);
if (string.IsNullOrEmpty(content))
{
return(string.Empty);
}
//var regex = new Regex(@"<img.*src=""(?<img>.*(wallpaper.*\.(jpg|png)))""");
var regex = new Regex(@"\+B\('(?<img>.*?)'\)");
var m = regex.Match(content);
if (m.Groups["img"].Success && !string.IsNullOrEmpty(m.Groups["img"].Value))
{
byte[] decoded = Convert.FromBase64String(m.Groups["img"].Value);
string final = Encoding.Default.GetString(decoded);
return(final);
}
return(string.Empty);
}
}
private string GetDirectPictureUrl(string pageUrl)
{
using (HttpUtility.CookieAwareWebClient cawc = new HttpUtility.CookieAwareWebClient(_cookies))
{
var content = cawc.DownloadString(pageUrl);
if (string.IsNullOrEmpty(content))
{
return(string.Empty);
}
//<img id="wallpaper" src="http://alpha.wallhaven.cc/wallpapers/full/wallhaven-4010.jpg" alt="" draggable="false" data-wallpaper-id="4010" data-wallpaper-width="1920" data-wallpaper-height="1200" class="fill-horizontal" style="margin-top: 0px; margin-bottom: 0px;">
var regex = new Regex(@"<img.*src=""(?<img>.*(wallpapers.*\.(jpg|png)))""", RegexOptions.Singleline);
//var regex = new Regex(@"\+B\('(?<img>.*?)'\)");
var m = regex.Match(content);
if (m.Groups["img"].Success && !string.IsNullOrEmpty(m.Groups["img"].Value))
{
return(m.Groups["img"].Value);
//byte[] decoded = Convert.FromBase64String(m.Groups["img"].Value);
//string final = Encoding.Default.GetString(decoded);
//return final;
}
return(string.Empty);
}
}
public PictureList GetPictures(PictureSearch ps)
{
WallbaseImageSearchSettings wiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new WallbaseImageSearchSettings() : WallbaseImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//if max picture count is 0, then no maximum, else specified max
var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue;
int pageSize = wiss.GetPageSize();
int pageIndex = ps.PageToRetrieve; //set page to retreive if one is specified
var imgFoundCount = 0;
//authenticate to wallbase
Authenticate(wiss.Username, wiss.Password);
var wallResults = new List <Picture>();
string areaURL = wiss.BuildURL();
//string postParams = wiss.GetPostParams(search);
do
{
//calculate page index. Random does not use pages, so for random just refresh with same url
string strPageNum = (pageIndex > 0 && wiss.SA != "random") || (wiss.SA == "toplist" || wiss.SA == "user/collection" || wiss.SA == "user/favorites") ? (pageIndex * pageSize).ToString() : "";
string pageURL = areaURL.Contains("{0}") ? string.Format(areaURL, strPageNum) : areaURL;
//string content = HttpPost(pageURL, postParams);
string content = string.Empty;
using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
{
try
{
//if random then don't post values
if (wiss.SA == "random")
{
content = _client.DownloadString(pageURL);
}
else
{
byte[] reqResult = _client.UploadValues(pageURL, wiss.GetPostParams());
content = System.Text.Encoding.Default.GetString(reqResult);
}
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("Failed to download search results from wallbase.cc, error: {0}", ex.ToString()), Log.LoggerLevels.Warnings);
}
}
if (string.IsNullOrEmpty(content))
{
break;
}
//parse html and get count
var pics = ParsePictures(content);
imgFoundCount = pics.Count();
//if we have an image ban list check for them
// doing this in the provider instead of picture manager
// ensures that our count does not go down if we have a max
if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
{
pics = (from c in pics where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
}
wallResults.AddRange(pics);
//increment page index so we can get the next set of images if they exist
pageIndex++;
} while (imgFoundCount > 0 && wallResults.Count < maxPictureCount && ps.PageToRetrieve == 0);
PictureList result = FetchPictures(wallResults, ps.PreviewOnly);
result.Pictures = result.Pictures.Take(maxPictureCount).ToList();
return(result);
}
public PictureList GetPictures(PictureSearch ps)
{
var result = new PictureList()
{
FetchDate = DateTime.Now
};
//load provider search settings
GoogleImageSearchSettings giss = GoogleImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//to help not have so many null checks, if no search settings provided then use default ones
if (giss == null)
{
giss = new GoogleImageSearchSettings();
}
//if search is empty, return now since we can't search without it
if (string.IsNullOrEmpty(giss.Query))
{
return(result);
}
var pageIndex = ps.PageToRetrieve; //set page to retrieve if one specified
var imgFoundCount = 0;
//if max picture count is 0, then no maximum, else specified max
var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue;
//build tbs strring
var tbs = "";//isz:ex,iszw:{1},iszh:{2}
//handle sizeing
if (giss.ImageHeight > 0 && giss.ImageWidth > 0)
{
tbs += string.Format("isz:ex,iszw:{0},iszh:{1},", giss.ImageWidth.ToString(), giss.ImageHeight.ToString());
}
//handle colors
if (!string.IsNullOrEmpty(giss.Color))
{
tbs += GoogleImageSearchSettings.GoogleImageColors.GetColorSearchString((from c in GoogleImageSearchSettings.GoogleImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ",";
}
//if we have a filter string then add it and trim off trailing commas
if (!string.IsNullOrEmpty(tbs))
{
tbs = ("&tbs=" + tbs).Trim(new char[] { ',' });
}
//do safe search setup (off/strict/moderate) this is part of the session and tracked via cookies
SetSafeSearchSetting(giss.GoogleSafeSearchOption);
do
{
//build URL from query, dimensions and page index
var url = string.Format(baseURL, giss.Query, tbs, (pageIndex * 20).ToString());
var response = string.Empty;
using (HttpUtility.CookieAwareWebClient client = new HttpUtility.CookieAwareWebClient(_cookies))
{
response = client.DownloadString(url);
}
var images = imagesRegex2.Matches(response);
//track number of images found for paging purposes
imgFoundCount = images.Count;
//convert images found into picture entries
foreach (Match item in images)
{
var purl = item.Groups["imgurlgrp"].Value;
var referrer = item.Groups["imgrefgrp"].Value;
var thumbnail = item.Groups["thumbURL"].Value;
//get id and trim if necessary (ran into a few cases of rediculously long filenames)
var id = System.IO.Path.GetFileNameWithoutExtension(purl);
if (id.Length > 50)
{
id = id.Substring(0, 50);
}
//because google images come from so many sites it's not uncommon to have duplicate file names. (we fix this)
id = string.Format("{0}_{1}", id, purl.GetHashCode().ToString());
Picture p = new Picture()
{
Url = purl, Id = id
};
p.Properties.Add(Picture.StandardProperties.Thumbnail, thumbnail);
p.Properties.Add(Picture.StandardProperties.Referrer, referrer);
result.Pictures.Add(p);
}
//if we have an image ban list check for them
// doing this in the provider instead of picture manager
// ensures that our count does not go down if we have a max
if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
{
result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
}
//increment page index so we can get the next 20 images if they exist
pageIndex++;
// Max Picture count is defined in search settings passed in, check for it here too
} while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount && ps.PageToRetrieve == 0);
result.Pictures = result.Pictures.Take(maxPictureCount).ToList();
return(result);
}