private void Form1_Load(object sender, EventArgs e)
{
WallbaseImageSearchSettings wiss = new WallbaseImageSearchSettings();
wiss.SA = "toplist";
PictureSearch ps = new PictureSearch();
ps.SearchProvider = new ActiveProviderInfo("Wallbase")
{
Active = true, ProviderConfig = wiss.Save()
};
ps.MaxPictureCount = 100;
string path = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "TEMP");
Directory.CreateDirectory(path);
ps.SaveFolder = path;
IInputProvider p = ps.SearchProvider.Instance as IInputProvider;
pl = p.GetPictures(ps);
Timer t = new Timer();
t.Tick += t_Tick;
t.Interval = 5000;
t.Enabled = true;
}
public PictureList GetPictures(PictureSearch ps)
{
PictureList pl = new PictureList() { FetchDate = DateTime.Now };
LocalDirectorySettings lds = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new LocalDirectorySettings() : LocalDirectorySettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//get all files in directory and filter for image extensions (jpg/jpeg/png/bmp?)
List<string> files = new List<string>();
foreach (string ext in lds.Extensions.Split(new char[] { ';' }))
{
files.AddRange(Directory.GetFiles(lds.Directory, "*." + ext, lds.IncludeSubdirectories?SearchOption.AllDirectories:SearchOption.TopDirectoryOnly));
}
//distinct list (just in case)
files = files.Distinct().ToList();
var maxPictureCount = ps.MaxPictureCount > 0 ? ps.MaxPictureCount : int.MaxValue;
maxPictureCount = Math.Min(files.Count, maxPictureCount);
//create picture items
pl.Pictures.AddRange((from c in files
select new Picture()
{
Id = Path.GetFileNameWithoutExtension(c),
Url=c,
LocalPath = c
})
.OrderBy(x=>Guid.NewGuid())
.Take(maxPictureCount));
return pl;
}
public async Task <IActionResult> OnPostAsync(int page, int rows, PictureSearch where)
{
var config = await _configService.GetAsync();
var data = await _service.GetListAsync(page, rows, where, null);
if (config.PictureUrl.IsNotNullOrEmpty())
{
foreach (var item in data.items)
{
item.Url = config.PictureUrl.Contact(item.FilePath);
item.ThumbUrl = config.PictureUrl.Contact(item.ThumbPath);
}
}
return(PagerData(data.items, page, rows, data.count));
}
private void button1_Click(object sender, EventArgs e)
{
PictureSearch ps = new PictureSearch();
ps.SearchProvider = new ActiveProviderInfo(providerComboBox1.SelectedItem.ToString()) { Active = true, ProviderConfig = _current.SaveConfiguration() };
ps.MaxPictureCount = (int)numericUpDown1.Value;
ps.SaveFolder = txtDownloadPath.Text;
IInputProvider p = ps.SearchProvider.Instance as IInputProvider;
var pl = p.GetPictures(ps);
PictureBatch pb = new PictureBatch();
pb.AllPictures.Add(pl);
DownloadManager.Current.SaveFolder = txtDownloadPath.Text;
DownloadManager.Current.PreFetchFiles(pb);
}
private void Form1_Load(object sender, EventArgs e)
{
WallbaseImageSearchSettings wiss = new WallbaseImageSearchSettings();
wiss.SA="toplist";
PictureSearch ps = new PictureSearch();
ps.SearchProvider = new ActiveProviderInfo("Wallbase") {Active=true, ProviderConfig = wiss.Save()};
ps.MaxPictureCount = 100;
string path = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "TEMP");
Directory.CreateDirectory(path);
ps.SaveFolder = path;
IInputProvider p = ps.SearchProvider.Instance as IInputProvider;
pl = p.GetPictures(ps);
Timer t = new Timer();
t.Tick += t_Tick;
t.Interval = 5000;
t.Enabled = true;
}
public Pagination <PictureDto> Execute(PictureSearch querry)
{
var query = Context.Pictures.AsQueryable();
if (querry.Keyword != null)
{
query = query.Where(p => p.src
.ToLower()
.Contains(querry.Keyword.ToLower()));
}
if (querry.Alt != null)
{
query = query.Where(p => p.alt
.ToLower()
.Contains(querry.Keyword.ToLower()));
}
var totalCount = query.Count();
query = query.Skip((querry.PageNumber - 1) * querry.PerPage).Take(querry.PerPage);
var pagesCount = (int)Math.Ceiling((double)totalCount / querry.PerPage);
return(new Pagination <PictureDto>
{
CurrentPage = querry.PageNumber,
Pages = pagesCount,
Total = totalCount,
Data = query.Include(p => p.Post)
.Select(p => new PictureDto
{
id = p.Id,
alt = p.alt,
src = p.src,
PostId = p.Post.Id,
CreatedAt = p.CreatedAt
})
});
}
private void button1_Click(object sender, EventArgs e)
{
PictureSearch ps = new PictureSearch();
ps.SearchProvider = new ActiveProviderInfo(providerComboBox1.SelectedItem.ToString())
{
Active = true, ProviderConfig = _current.SaveConfiguration()
};
ps.MaxPictureCount = (int)numericUpDown1.Value;
ps.SaveFolder = txtDownloadPath.Text;
IInputProvider p = ps.SearchProvider.Instance as IInputProvider;
var pl = p.GetPictures(ps);
PictureBatch pb = new PictureBatch();
pb.AllPictures.Add(pl);
DownloadManager.Current.SaveFolder = txtDownloadPath.Text;
DownloadManager.Current.PreFetchFiles(pb);
}
void DownloadNextPicture()
{
var ps = new PictureSearch()
{
SearchString = App.Settings.Search,
PreFetch = App.Settings.PreFetch,
SaveFolder = App.Settings.CachePath,
MaxPictureCount = App.Settings.MaxPictureDownloadCount,
SearchProvider = App.CurrentProvider,
BannedURLs = App.Settings.BannedImages,
ProviderSearchSettings = App.Settings.ProviderSettings.ContainsKey(App.Settings.Provider) ? App.Settings.ProviderSettings[App.Settings.Provider].ProviderConfig : string.Empty
};
if (App.Settings.Language == "ru-RU" || App.Settings.Provider != "Rewalls")
{
App.PictureManager.GetPicture(ps);
}
else
{
if (string.IsNullOrEmpty(translatedKeyword))
{
ThreadStart threadStarter = delegate
{
translatedKeyword = Translator.TranslateText(App.Settings.Search, "en|ru");
ps.SearchString = translatedKeyword;
App.PictureManager.GetPicture(ps);
};
var thread = new Thread(threadStarter);
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
}
else
{
ps.SearchString = translatedKeyword;
App.PictureManager.GetPicture(ps);
}
}
}
public PictureList GetPictures(PictureSearch ps)
{
PictureList pl = new PictureList()
{
FetchDate = DateTime.Now
};
LocalDirectorySettings lds = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new LocalDirectorySettings() : LocalDirectorySettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//get all files in directory and filter for image extensions (jpg/jpeg/png/bmp?)
List <string> files = new List <string>();
foreach (string ext in lds.Extensions.Split(new char[] { ';' }))
{
files.AddRange(Directory.GetFiles(lds.Directory, "*." + ext, SearchOption.AllDirectories));
}
//distinct list (just in case)
files = files.Distinct().ToList();
var maxPictureCount = ps.MaxPictureCount > 0 ? ps.MaxPictureCount : int.MaxValue;
maxPictureCount = Math.Min(files.Count, maxPictureCount);
//create picture items
pl.Pictures.AddRange((from c in files
select new Picture()
{
Id = Path.GetFileNameWithoutExtension(c),
Url = c,
LocalPath = c
})
.OrderBy(x => Guid.NewGuid())
.Take(maxPictureCount));
return(pl);
}
protected void DownloadNextPicture()
{
if (CurrentInputProviders.Count == 0) return;
//create the new picture batch
PictureBatch pb = new PictureBatch() {PreviousBatch = CurrentBatch};
//create another view of the input providers, otherwise if the list changes
// because user changes options then it breaks :)
foreach (KeyValuePair<Guid, ActiveProviderInfo> kvpGAPI in CurrentInputProviders.ToArray())
{
ActiveProviderInfo api = kvpGAPI.Value;
var ps = new PictureSearch()
{
SaveFolder = Settings.CurrentSettings.CachePath,
MaxPictureCount = Settings.CurrentSettings.MaxPictureDownloadCount,
SearchProvider = api,
BannedURLs = Settings.CurrentSettings.BannedImages
};
//get new pictures
PictureList pl = PictureManager.GetPictureList(ps);
//save to picturebatch
pb.AllPictures.Add(pl);
}
//process downloaded picture list
ProcessDownloadedPicture(pb);
//if prefetch is enabled, validate that all pictures have been downloaded
if (Settings.CurrentSettings.PreFetch) DownloadManager.PreFetchFiles(pb);
}
public PictureList LoadCachedSearch(PictureSearch ps, string cachePath)
{
PictureList result = null;
//check if we should load from file
if (File.Exists(cachePath))
{
try
{
result = PictureList.LoadFromFile(cachePath);
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("Error loading picture cache from file, cache will not be used. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors);
}
}
return result;
}
public PictureList GetPictures(PictureSearch ps)
{
WallbaseImageSearchSettings wiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new WallbaseImageSearchSettings() : WallbaseImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//if max picture count is 0, then no maximum, else specified max
var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue;
int pageSize = wiss.GetPageSize();
int pageIndex = ps.PageToRetrieve; //set page to retreive if one is specified
var imgFoundCount = 0;
//authenticate to wallbase
Authenticate(wiss.Username, wiss.Password);
var wallResults = new List <Picture>();
string areaURL = wiss.BuildURL();
//string postParams = wiss.GetPostParams(search);
do
{
//calculate page index. Random does not use pages, so for random just refresh with same url
string strPageNum = (pageIndex > 0 && wiss.SA != "random") || (wiss.SA == "toplist" || wiss.SA == "user/collection" || wiss.SA == "user/favorites") ? (pageIndex * pageSize).ToString() : "";
string pageURL = areaURL.Contains("{0}") ? string.Format(areaURL, strPageNum) : areaURL;
//string content = HttpPost(pageURL, postParams);
string content = string.Empty;
using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
{
try
{
//if random then don't post values
if (wiss.SA == "random")
{
content = _client.DownloadString(pageURL);
}
else
{
byte[] reqResult = _client.UploadValues(pageURL, wiss.GetPostParams());
content = System.Text.Encoding.Default.GetString(reqResult);
}
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("Failed to download search results from wallbase.cc, error: {0}", ex.ToString()), Log.LoggerLevels.Warnings);
}
}
if (string.IsNullOrEmpty(content))
{
break;
}
//parse html and get count
var pics = ParsePictures(content);
imgFoundCount = pics.Count();
//if we have an image ban list check for them
// doing this in the provider instead of picture manager
// ensures that our count does not go down if we have a max
if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
{
pics = (from c in pics where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
}
wallResults.AddRange(pics);
//increment page index so we can get the next set of images if they exist
pageIndex++;
} while (imgFoundCount > 0 && wallResults.Count < maxPictureCount && ps.PageToRetrieve == 0);
PictureList result = FetchPictures(wallResults, ps.PreviewOnly);
result.Pictures = result.Pictures.Take(maxPictureCount).ToList();
return(result);
}
public async Task <IActionResult> OnPostDataAsync(int page, int rows, PictureSearch where)
{
var data = await _service.GetListAsync(page, rows, where, null);
return(PagerData(data.items, page, rows, data.count));
}
public PictureList GetPictureList(PictureSearch ps)
{
PictureList Pictures = null;
if (ps == null || ps.SearchProvider == null || ps.SearchProvider.Instance == null) return Pictures;
//load any in memory cached results
Pictures = ps.SearchProvider.SearchResults;
var loadedFromFile = false;
var fPath = Path.Combine(ps.SaveFolder, "CACHE_" + ps.GetSearchHash().ToString() + "_" + ps.SearchProvider.Instance.GetType().ToString() + ".xml");
if (Pictures == null)
{
//if nothing in memory then try to load from disc
Pictures = LoadCachedSearch(ps, fPath);
loadedFromFile = Pictures != null;
}
else
{
loadedFromFile = false;
}
//if we have no pictures to work with, or our cached data has expired, try and get them
if (Pictures == null || Pictures.Pictures.Count == 0 || Pictures.ExpirationDate < DateTime.Now)
{
Pictures = ((IInputProvider)ps.SearchProvider.Instance).GetPictures(ps);
Pictures.SearchSettingsHash = ps.GetSearchHash();
loadedFromFile = false;
}
//cache the picture list to file
if (!loadedFromFile)
{
//make sure the API GuID has been injected into all pictures
Pictures.Pictures.ForEach(x => x.ProviderInstance = ps.SearchProvider.ProviderInstanceID);
//save it
Pictures.Save(fPath);
}
//return whatever list of pictures was found
return Pictures;
}
public IActionResult Get([FromQuery] PictureSearch search, [FromServices] IGetPicturesQuery query)
{
return(Ok(executor.ExecuteQuery(query, search)));
}
public PictureList GetPictures(PictureSearch ps)
{
WallbaseImageSearchSettings wiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ? new WallbaseImageSearchSettings() : WallbaseImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//if max picture count is 0, then no maximum, else specified max
var maxPictureCount = wiss.GetMaxImageCount(ps.MaxPictureCount);
int pageSize = wiss.GetPageSize();
int pageIndex = ps.PageToRetrieve; //set page to retreive if one is specified
var imgFoundCount = 0;
//authenticate to wallbase
Authenticate(wiss.Username, wiss.Password);
var wallResults = new List<Picture>();
string areaURL = wiss.BuildURL();
do
{
//calculate page index. Random does not use pages, so for random just refresh with same url
string strPageNum = (pageIndex * pageSize).ToString();
string pageURL = areaURL.Contains("{0}") ? string.Format(areaURL, strPageNum) : areaURL;
//string content = HttpPost(pageURL, postParams);
string content = string.Empty;
using (HttpUtility.CookieAwareWebClient _client = new HttpUtility.CookieAwareWebClient(_cookies))
{
try
{
//if random then don't post values
//if (wiss.SA == "random")
//{
content = _client.DownloadString(pageURL);
//}
//else
//{
// byte[] reqResult = _client.UploadValues(pageURL, wiss.GetPostParams());
// content = System.Text.Encoding.Default.GetString(reqResult);
//}
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("Failed to download search results from wallbase.cc, error: {0}", ex.ToString()), Log.LoggerLevels.Warnings);
}
}
if (string.IsNullOrEmpty(content))
break;
//parse html and get count
var pics = ParsePictures(content);
imgFoundCount = pics.Count();
//if we have an image ban list check for them
// doing this in the provider instead of picture manager
// ensures that our count does not go down if we have a max
if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
{
pics = (from c in pics where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
}
wallResults.AddRange(pics);
//increment page index so we can get the next set of images if they exist
pageIndex++;
} while (imgFoundCount > 0 && wallResults.Count < maxPictureCount && ps.PageToRetrieve == 0);
PictureList result = FetchPictures(wallResults, ps.PreviewOnly);
result.Pictures = result.Pictures.Take(maxPictureCount).ToList();
return result;
}
public PictureList GetPictures(PictureSearch ps)
{
var result = new PictureList()
{
FetchDate = DateTime.Now
};
//load provider search settings
GoogleImageSearchSettings giss = GoogleImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
//to help not have so many null checks, if no search settings provided then use default ones
if (giss == null)
{
giss = new GoogleImageSearchSettings();
}
//if search is empty, return now since we can't search without it
if (string.IsNullOrEmpty(giss.Query))
{
return(result);
}
var pageIndex = ps.PageToRetrieve; //set page to retrieve if one specified
var imgFoundCount = 0;
//if max picture count is 0, then no maximum, else specified max
var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue;
//build tbs strring
var tbs = "";//isz:ex,iszw:{1},iszh:{2}
//handle sizeing
if (giss.ImageHeight > 0 && giss.ImageWidth > 0)
{
tbs += string.Format("isz:ex,iszw:{0},iszh:{1},", giss.ImageWidth.ToString(), giss.ImageHeight.ToString());
}
//handle colors
if (!string.IsNullOrEmpty(giss.Color))
{
tbs += GoogleImageSearchSettings.GoogleImageColors.GetColorSearchString((from c in GoogleImageSearchSettings.GoogleImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ",";
}
//if we have a filter string then add it and trim off trailing commas
if (!string.IsNullOrEmpty(tbs))
{
tbs = ("&tbs=" + tbs).Trim(new char[] { ',' });
}
//do safe search setup (off/strict/moderate) this is part of the session and tracked via cookies
SetSafeSearchSetting(giss.GoogleSafeSearchOption);
do
{
//build URL from query, dimensions and page index
var url = string.Format(baseURL, giss.Query, tbs, (pageIndex * 20).ToString());
var response = string.Empty;
using (HttpUtility.CookieAwareWebClient client = new HttpUtility.CookieAwareWebClient(_cookies))
{
response = client.DownloadString(url);
}
var images = imagesRegex2.Matches(response);
//track number of images found for paging purposes
imgFoundCount = images.Count;
//convert images found into picture entries
foreach (Match item in images)
{
var purl = item.Groups["imgurlgrp"].Value;
var referrer = item.Groups["imgrefgrp"].Value;
var thumbnail = item.Groups["thumbURL"].Value;
//get id and trim if necessary (ran into a few cases of rediculously long filenames)
var id = System.IO.Path.GetFileNameWithoutExtension(purl);
if (id.Length > 50)
{
id = id.Substring(0, 50);
}
//because google images come from so many sites it's not uncommon to have duplicate file names. (we fix this)
id = string.Format("{0}_{1}", id, purl.GetHashCode().ToString());
Picture p = new Picture()
{
Url = purl, Id = id
};
p.Properties.Add(Picture.StandardProperties.Thumbnail, thumbnail);
p.Properties.Add(Picture.StandardProperties.Referrer, referrer);
result.Pictures.Add(p);
}
//if we have an image ban list check for them
// doing this in the provider instead of picture manager
// ensures that our count does not go down if we have a max
if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
{
result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
}
//increment page index so we can get the next 20 images if they exist
pageIndex++;
// Max Picture count is defined in search settings passed in, check for it here too
} while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount && ps.PageToRetrieve == 0);
result.Pictures = result.Pictures.Take(maxPictureCount).ToList();
return(result);
}
public IActionResult Get([FromQuery] PictureSearch dto, [FromServices] IGetPicturesCommand query) => Ok(executor.ExecuteQuery(query, dto));
public PictureList GetPictures(PictureSearch ps)
{
var result = new PictureList() { FetchDate = DateTime.Now };
MediaRSSImageSearchSettings mrssiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ?
new MediaRSSImageSearchSettings() : MediaRSSImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
XDocument feedXML = XDocument.Load(mrssiss.MediaRSSURL);
XNamespace media = XNamespace.Get("http://search.yahoo.com/mrss/");
var feeds = from feed in feedXML.Descendants("item")
let content = feed.Elements(media + "content")
let thumb = feed.Element(media + "thumbnail").Attribute("url").Value
let img = content.Where(x => x.Attribute("medium").Value == "image").SingleOrDefault()
let url = img!=null?img.Attribute("url").Value:thumb
let id = System.IO.Path.GetFileNameWithoutExtension(url)
select new Picture()
{
Url = url,
Id = id.Length > 50 ? id.Substring(0, 50) : id,
Properties = new SerializableDictionary<string,string>(Picture.StandardProperties.Thumbnail,thumb)
};
//get up to the maximum number of pictures, excluding banned images
result.Pictures.AddRange(
feeds.Where(x=> !ps.BannedURLs.Contains(x.Url))
.Take(ps.MaxPictureCount));
////handle colors
//if (!string.IsNullOrEmpty(giss.Color))
//{
// tbs += MediaRSSImageSearchSettings.MediaRSSImageColors.GetColorSearchString((from c in MediaRSSImageSearchSettings.MediaRSSImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ",";
//}
////if we have a filter string then add it and trim off trailing commas
//if (!string.IsNullOrEmpty(tbs)) tbs = ("&tbs=" + tbs).Trim(new char[]{','});
//do
//{
// //build URL from query, dimensions and page index
// var url = string.Format(baseURL, ps.SearchString, tbs, (pageIndex * 20).ToString());
// var response = client.DownloadString(url);
// var images = imagesRegex2.Matches(response);
// //track number of images found for paging purposes
// imgFoundCount = images.Count;
// //convert images found into picture entries
// foreach (Match item in images)
// {
// var purl = item.Groups[3].Value;
// //get id and trim if necessary (ran into a few cases of rediculously long filenames)
// var id = System.IO.Path.GetFileNameWithoutExtension(purl);
// if (id.Length > 50) id = id.Substring(0, 50);
// result.Pictures.Add(new Picture() { Url = purl, Id = id });
// }
// //if we have an image ban list check for them
// // doing this in the provider instead of picture manager
// // ensures that our count does not go down if we have a max
// if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
// {
// result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
// }
// //increment page index so we can get the next 20 images if they exist
// pageIndex++;
// // Max Picture count is defined in search settings passed in, check for it here too
//} while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount);
return result;
}
public PictureList GetPictures(PictureSearch ps)
{
var result = new PictureList()
{
FetchDate = DateTime.Now
};
MediaRSSImageSearchSettings mrssiss = string.IsNullOrEmpty(ps.SearchProvider.ProviderConfig) ?
new MediaRSSImageSearchSettings() : MediaRSSImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig);
XDocument feedXML = XDocument.Load(mrssiss.MediaRSSURL);
XNamespace media = XNamespace.Get("http://search.yahoo.com/mrss/");
var feeds = from feed in feedXML.Descendants("item")
let content = feed.Elements(media + "content")
let thumb = feed.Element(media + "thumbnail").Attribute("url").Value
let img = content.Where(x => x.Attribute("medium").Value == "image").SingleOrDefault()
let url = img != null?img.Attribute("url").Value : thumb
let id = System.IO.Path.GetFileNameWithoutExtension(url)
select new Picture()
{
Url = url,
Id = id.Length > 50 ? id.Substring(0, 50) : id,
Properties = new SerializableDictionary <string, string>(Picture.StandardProperties.Thumbnail, thumb)
};
//get up to the maximum number of pictures, excluding banned images
result.Pictures.AddRange(
feeds.Where(x => !ps.BannedURLs.Contains(x.Url))
.Take(ps.MaxPictureCount));
////handle colors
//if (!string.IsNullOrEmpty(giss.Color))
//{
// tbs += MediaRSSImageSearchSettings.MediaRSSImageColors.GetColorSearchString((from c in MediaRSSImageSearchSettings.MediaRSSImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ",";
//}
////if we have a filter string then add it and trim off trailing commas
//if (!string.IsNullOrEmpty(tbs)) tbs = ("&tbs=" + tbs).Trim(new char[]{','});
//do
//{
// //build URL from query, dimensions and page index
// var url = string.Format(baseURL, ps.SearchString, tbs, (pageIndex * 20).ToString());
// var response = client.DownloadString(url);
// var images = imagesRegex2.Matches(response);
// //track number of images found for paging purposes
// imgFoundCount = images.Count;
// //convert images found into picture entries
// foreach (Match item in images)
// {
// var purl = item.Groups[3].Value;
// //get id and trim if necessary (ran into a few cases of rediculously long filenames)
// var id = System.IO.Path.GetFileNameWithoutExtension(purl);
// if (id.Length > 50) id = id.Substring(0, 50);
// result.Pictures.Add(new Picture() { Url = purl, Id = id });
// }
// //if we have an image ban list check for them
// // doing this in the provider instead of picture manager
// // ensures that our count does not go down if we have a max
// if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
// {
// result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
// }
// //increment page index so we can get the next 20 images if they exist
// pageIndex++;
// // Max Picture count is defined in search settings passed in, check for it here too
//} while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount);
return(result);
}
public PictureList GetPictures(PictureSearch ps)
{
var result = new PictureList() { FetchDate = DateTime.Now };
//load provider search settings
GoogleImageSearchSettings giss = GoogleImageSearchSettings.LoadFromXML(ps.SearchProvider.ProviderConfig) ??
new GoogleImageSearchSettings();
//if search is empty, return now since we can't search without it
if (string.IsNullOrEmpty(giss.Query)) return result;
var pageIndex = ps.PageToRetrieve; //set page to retrieve if one specified
var imgFoundCount = 0;
//if max picture count is 0, then no maximum, else specified max
var maxPictureCount = ps.MaxPictureCount > 0?ps.MaxPictureCount : int.MaxValue;
//build tbs strring
var tbs = "";//isz:ex,iszw:{1},iszh:{2}
//handle sizeing
if (giss.ImageHeight > 0 && giss.ImageWidth > 0)
{
tbs += string.Format("isz:ex,iszw:{0},iszh:{1},", giss.ImageWidth, giss.ImageHeight);
}
//handle colors
if (!string.IsNullOrEmpty(giss.Color))
{
tbs += GoogleImageSearchSettings.GoogleImageColors.GetColorSearchString((from c in GoogleImageSearchSettings.GoogleImageColors.GetColors() where c.Value == giss.Color select c).Single()) + ",";
}
//if we have a filter string then add it and trim off trailing commas
if (!string.IsNullOrEmpty(tbs)) tbs = ("&tbs=" + tbs).Trim(new char[]{','});
//do safe search setup (off/strict/moderate) this is part of the session and tracked via cookies
//SetSafeSearchSetting(giss.GoogleSafeSearchOption);
do
{
//build URL from query, dimensions and page index
var url = string.Format(baseURL, giss.Query, tbs, (pageIndex * 20));
var response = string.Empty;
using (var client = new HttpUtility.CookieAwareWebClient(_cookies))
{
response = client.DownloadString(url);
}
var images = _imagesRegex2.Matches(response);
//track number of images found for paging purposes
imgFoundCount = images.Count;
//convert images found into picture entries
foreach (Match item in images)
{
var purl = item.Groups["imgurlgrp"].Value;
var referrer = item.Groups["imgrefgrp"].Value;
var thumbnail = item.Groups["thumbURL"].Value;
//get id and trim if necessary (ran into a few cases of rediculously long filenames)
var id = System.IO.Path.GetFileNameWithoutExtension(purl);
if (id.Length > 50) id = id.Substring(0, 50);
//because google images come from so many sites it's not uncommon to have duplicate file names. (we fix this)
id = string.Format("{0}_{1}", id, purl.GetHashCode());
var p = new Picture() { Url = purl, Id = id };
p.Properties.Add(Picture.StandardProperties.Thumbnail, thumbnail);
p.Properties.Add(Picture.StandardProperties.Referrer, referrer);
result.Pictures.Add(p);
}
//if we have an image ban list check for them
// doing this in the provider instead of picture manager
// ensures that our count does not go down if we have a max
if (ps.BannedURLs != null && ps.BannedURLs.Count > 0)
{
result.Pictures = (from c in result.Pictures where !(ps.BannedURLs.Contains(c.Url)) select c).ToList();
}
//increment page index so we can get the next 20 images if they exist
pageIndex++;
// Max Picture count is defined in search settings passed in, check for it here too
} while (imgFoundCount > 0 && result.Pictures.Count < maxPictureCount && ps.PageToRetrieve == 0);
result.Pictures = result.Pictures.Take(maxPictureCount).ToList();
return result;
}
