private Datamodels.TumblrJson RequestData(string url, string authHeaders)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.ServicePoint.Expect100Continue = false;
request.ContentType = "x-www-from-urlencoded";
if (authHeaders != null)
{
// add OAuth header
request.Headers["Authorization"] = authHeaders;
}
var jsserializer = new JavaScriptSerializer();
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
{
ThrottledStream stream = new ThrottledStream(response.GetResponseStream(), (shellService.Settings.Bandwidth / shellService.Settings.ParallelImages) * 1024);
StreamReader reader = new StreamReader(stream);
DataModels.TumblrJson data = jsserializer.Deserialize <DataModels.TumblrJson>(reader.ReadToEnd());
if (data.meta.status == 200)
{
return(data);
}
return(null);
}
}
catch (WebException ex)
{
return(null);
}
}
public Tuple <uint, List <string> > GetImageUrls(TumblrBlog blog, IProgress <DataModels.DownloadProgress> progress, CancellationToken ct, PauseToken pt)
{
int totalPosts = 0;
int numberOfPostsCrawled = 0;
uint totalImages;
List <string> images = new List <string>();
string url = GetApiUrl(blog.Name, 1);
string authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
var blogDoc = RequestData(url, authHeader);
totalPosts = blogDoc.response.blog.total_posts;
// Generate URL list of Images
// the api v2 shows 20 posts at max, determine the number of pages to crawl
int totalPages = (totalPosts / 20) + 1;
Parallel.For(0, totalPages,
new ParallelOptions {
MaxDegreeOfParallelism = (shellService.Settings.ParallelImages / selectionService.ActiveItems.Count)
},
(i, state) =>
{
if (ct.IsCancellationRequested)
{
state.Break();
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
try
{
// check for tags -- crawling for all images here
if (blog.Tags == null || blog.Tags.Count() == 0)
{
DataModels.TumblrJson document = null;
// get 20 posts per crawl/page
url = GetApiUrl(blog.Name, 20, i * 20);
authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
document = RequestData(url, authHeader);
if (shellService.Settings.DownloadImages == true)
{
foreach (Datamodels.Post post in document.response.posts.Where(posts => posts.type.Equals("photo")))
{
foreach (DataModels.Photo photo in post.photos)
{
var imageUrl = photo.alt_sizes.ElementAt(shellService.Settings.ImageSizes.IndexOf(shellService.Settings.ImageSize.ToString())).url;
if (shellService.Settings.SkipGif == true && imageUrl.EndsWith(".gif"))
{
continue;
}
Monitor.Enter(images);
images.Add(imageUrl);
Monitor.Exit(images);
}
}
}
if (shellService.Settings.DownloadVideos == true)
{
foreach (DataModels.Post post in document.response.posts.Where(posts => posts.type.Equals("video")))
{
if (shellService.Settings.VideoSize == 1080)
{
Monitor.Enter(images);
images.Add(post.video_url);
Monitor.Exit(images);
}
if (shellService.Settings.VideoSize == 480)
{
Monitor.Enter(images);
images.Add(post.video_url.Insert(post.video_url.LastIndexOf("."), "_480"));
Monitor.Exit(images);
}
}
}
}
// crawling only for tagged images
else
{
List <string> tags = blog.Tags.Split(',').Select(x => x.Trim()).ToList();
DataModels.TumblrJson document = null;
// get 20 posts per crawl/page
url = GetApiUrl(blog.Name, 20, i * 20);
authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
document = RequestData(url, authHeader);
if (shellService.Settings.DownloadImages == true)
{
foreach (Datamodels.Post post in document.response.posts.Where(posts => posts.tags.Any(tag => tags.Equals(tag)) && posts.type.Equals("photo")))
{
foreach (DataModels.Photo photo in post.photos ?? new List <Datamodels.Photo>())
{
var imageUrl = photo.alt_sizes.ElementAt(shellService.Settings.ImageSizes.IndexOf(shellService.Settings.ImageSize.ToString())).url;
if (shellService.Settings.SkipGif == true && imageUrl.EndsWith(".gif"))
{
continue;
}
Monitor.Enter(images);
images.Add(imageUrl);
Monitor.Exit(images);
}
}
}
if (shellService.Settings.DownloadVideos == true)
{
foreach (DataModels.Post post in document.response.posts.Where(posts => posts.tags.Any(tag => tags.Equals(tag)) && posts.type.Equals("video")))
{
if (shellService.Settings.VideoSize == 1080)
{
Monitor.Enter(images);
images.Add(post.video_url);
Monitor.Exit(images);
}
if (shellService.Settings.VideoSize == 480)
{
Monitor.Enter(images);
images.Add(post.video_url.Insert(post.video_url.LastIndexOf("."), "_480"));
Monitor.Exit(images);
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Data);
}
numberOfPostsCrawled += 20;
var newProgress = new DataModels.DownloadProgress();
newProgress.Progress = string.Format(CultureInfo.CurrentCulture, Resources.ProgressGetUrl, numberOfPostsCrawled, totalPosts);
progress.Report(newProgress);
}
);
images = images.Distinct().ToList();
totalImages = (uint)images.Count;
return(Tuple.Create(totalImages, images));
}
