private async Task AddBlogAsync(string blogUrl)
{
if (string.IsNullOrEmpty(blogUrl))
{
blogUrl = crawlerService.NewBlogUrl;
}
// FIXME: Dependency
var blog = new TumblrBlog(blogUrl, Path.Combine(shellService.Settings.DownloadLocation, "Index"), BlogTypes.tumblr);
TransferGlobalSettingsToBlog(blog);
IDownloader downloader = DownloaderFactory.GetDownloader(blog.BlogType, shellService, crawlerService, blog);
await downloader.IsBlogOnlineAsync();
await downloader.UpdateMetaInformationAsync();
lock (lockObject)
{
if (managerService.BlogFiles.Select(blogs => blogs.Name).ToList().Contains(blog.Name))
{
shellService.ShowError(null, Resources.BlogAlreadyExist, blog.Name);
return;
}
if (blog.Save())
{
QueueOnDispatcher.CheckBeginInvokeOnUI((Action)(() => managerService.BlogFiles.Add(blog)));
}
}
}
public TumblrDownloader(IShellService shellService, ICrawlerService crawlerService, IBlog blog)
: base(shellService, crawlerService, blog)
{
this.shellService = shellService;
this.crawlerService = crawlerService;
this.blog = (TumblrBlog)blog;
}
private TumblrBlog CrawlCoreTumblrBlog(TumblrBlog blog, IProgress <DataModels.DownloadProgress> progress, CancellationToken ct, PauseToken pt)
{
Logger.Verbose("ManagerController.CrawlCoreTumblrBlog:Start");
var newProgress = new DataModels.DownloadProgress();
var tuple = GetImageUrls(blog, progress, ct, pt);
var newImageCount = tuple.Item1;
var newImageUrls = tuple.Item2;
blog.TotalCount = newImageCount;
var imageUrls = newImageUrls.Except(blog.Links.ToList());
var indexPath = Path.Combine(shellService.Settings.DownloadLocation, "Index");
var blogPath = shellService.Settings.DownloadLocation;
var parallel = Parallel.ForEach(
imageUrls,
new ParallelOptions {
MaxDegreeOfParallelism = (shellService.Settings.ParallelImages / selectionService.ActiveItems.Count)
},
(currentImageUrl, state) =>
{
if (ct.IsCancellationRequested)
{
state.Break();
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
string fileName = currentImageUrl.Split('/').Last();
string fileLocation = Path.Combine(Path.Combine(blogPath, blog.Name), fileName);
if (Download(blog, fileLocation, currentImageUrl))
{
blog.Links.Add(currentImageUrl);
blog.DownloadedImages = (uint)blog.Links.Count();
blog.Progress = (uint)((double)blog.DownloadedImages / (double)blog.TotalCount * 100);
newProgress = new DataModels.DownloadProgress();
newProgress.Progress = string.Format(CultureInfo.CurrentCulture, Resources.ProgressDownloadImage, currentImageUrl);;
progress.Report(newProgress);
}
});
if (!ct.IsCancellationRequested)
{
blog.LastCompleteCrawl = DateTime.Now;
}
SaveBlog(blog);
newProgress = new DataModels.DownloadProgress();
newProgress.Progress = "";
progress.Report(newProgress);
return(blog);
}
private IReadOnlyList <Blog> GetFilesCore(string directory)
{
Logger.Verbose("ManagerController.UpdateBlogFiles:GetFilesAsync Start");
List <Blog> blogs = new List <Blog>();
foreach (var filename in Directory.GetFiles(directory))
{
using (FileStream stream = new FileStream(filename,
FileMode.Open, FileAccess.Read, FileShare.Read))
{
IFormatter formatter = new BinaryFormatter();
try
{
TumblrBlog blog = (TumblrBlog)formatter.Deserialize(stream);
blogs.Add(blog);
}
catch (SerializationException ex)
{
ex.Data["Filename"] = filename;
throw;
}
}
}
Logger.Verbose("ManagerController.UpdateBlogFiles:GetFilesAsync End");
return(blogs);
}
/// <summary>
///
/// </summary>
/// <param name="blog"></param>
/// <param name="url"></param>
/// <param name="apiMode"></param>
/// <param name="offset"></param>
/// <param name="limit"></param>
public TumblrStatsManager(TumblrBlog blog = null, string url = null, TumblrApiVersion apiMode = TumblrApiVersion.V2Json, int offset = 0, int limit = 0)
{
DocumentManager = new RemoteDocumentManager();
ApiVersion = apiMode;
DocumentManager.ApiVersion = apiMode;
TotalPostsPerDocument = (int)NumberOfPostsPerApiDocument.ApiV2;
Blog = blog ?? ((url != null) ? new TumblrBlog(url) : null);
if (Blog != null)
{
Blog.Posts = new HashSet <TumblrPost>();
TumblrUrl = WebHelper.RemoveTrailingBackslash(Blog.Url);
TumblrDomain = WebHelper.GetDomainName(TumblrUrl);
ApiQueryLimit = limit;
ApiQueryOffset = offset;
TotalPostsPerDocument = (int)NumberOfPostsPerApiDocument.ApiV2; //20 for JSON, 50 for XML
var values = Enum.GetValues(typeof(TumblrPostType)).Cast <TumblrPostType>();
TypesCount = values.Count() - 3;
// Get Blog Info
DocumentManager.GetRemoteBlogInfo(TumblrApiHelper.GeneratePostTypeQueryUrl(TumblrDomain, TumblrPostType.All, 0, 1), Blog);
}
}
public TagScanManager(TumblrBlog blog = null, bool photoPostOnly = false)
{
Blog = blog;
DocumentManager = new RemoteDocumentManager();
TumblrUrl = WebHelper.RemoveTrailingBackslash(blog.Url);
TumblrDomain = WebHelper.GetDomainName(blog.Url);
TagList = new HashSet <string>();
PhotoPostOnly = photoPostOnly;
}
public void TestMethod1()
{
TumblrBlog Blog = new TumblrBlog("http://jai-envie-de-toi.ml");
TagScanManager scanner = new TagScanManager(Blog);
scanner.GetTumblrBlogInfo();
scanner.ScanTags();
HashSet <string> s = scanner.TagList;
}
private async Task <IBlog> CheckIfCrawlableBlog(string blogUrl)
{
if (!_blogFactory.IsValidTumblrBlogUrl(blogUrl) && _blogFactory.IsValidUrl(blogUrl))
{
if (await _tumblrBlogDetector.IsTumblrBlogWithCustomDomainAsync(blogUrl))
{
return(TumblrBlog.Create(blogUrl, Path.Combine(_shellService.Settings.DownloadLocation, "Index"), _shellService.Settings.FilenameTemplate, true));
}
throw new Exception($"The url '{blogUrl}' cannot be recognized as Tumblr blog!");
}
return(_blogFactory.GetBlog(blogUrl, Path.Combine(_shellService.Settings.DownloadLocation, "Index"), _shellService.Settings.FilenameTemplate));
}
/// <summary>
///
/// </summary>
/// <param name="url"></param>
/// <param name="blog"></param>
/// <returns></returns>
public bool GetRemoteBlogInfo(string url, TumblrBlog blog)
{
if (!string.IsNullOrEmpty(url))
{
try
{
if (blog == null)
{
blog = new TumblrBlog(url);
}
dynamic jsonDocument = JsonHelper.GetDynamicObjectFromString(WebHelper.GetRemoteDocumentAsString(url));
if (jsonDocument != null && jsonDocument.response != null && jsonDocument.response.blog != null)
{
blog.Title = jsonDocument.response.blog.title;
blog.Description = jsonDocument.response.blog.description;
blog.Name = jsonDocument.response.blog.name;
blog.Url = jsonDocument.response.blog.url;
blog.Nsfw = Convert.ToBoolean(jsonDocument.response.blog.is_nsfw);
blog.AskEnabled = Convert.ToBoolean(jsonDocument.response.blog.ask);
blog.AnonAskEnabled = Convert.ToBoolean(jsonDocument.response.blog.ask_anon);
blog.LastUpdated = DateTimeHelper.UnixTimeStampToDateTime(Convert.ToDouble(jsonDocument.response.blog.updated));
if (jsonDocument.response.total_posts != null)
{
blog.TotalPosts = Convert.ToInt32(jsonDocument.response.total_posts);
}
else if (jsonDocument.response.blog.posts != null)
{
blog.TotalPosts = Convert.ToInt32(jsonDocument.response.blog.posts);
}
if (jsonDocument.response.blog.posts != null)
{
blog.BlogTotalPosts = Convert.ToInt32(jsonDocument.response.blog.posts);
}
return(true);
}
}
catch
{
return(false);
}
}
return(false);
}
private async Task <IBlog> CheckIfCrawlableBlog(string blogUrl, bool fromClipboard)
{
if (!_blogFactory.IsValidBlogUrl(blogUrl))
{
if (fromClipboard)
{
throw new Exception();
}
if (_blogFactory.IsValidUrl(blogUrl) && await _tumblrBlogDetector.IsTumblrBlogWithCustomDomainAsync(blogUrl))
{
return(TumblrBlog.Create(blogUrl, GetIndexFolderPath(_shellService.Settings.ActiveCollectionId), _shellService.Settings.FilenameTemplate, true));
}
throw new Exception($"The url '{blogUrl}' cannot be recognized as valid blog!");
}
return(_blogFactory.GetBlog(blogUrl, GetIndexFolderPath(_shellService.Settings.ActiveCollectionId), _shellService.Settings.FilenameTemplate));
}
private void TransferGlobalSettingsToBlog(TumblrBlog blog)
{
blog.DownloadAudio = shellService.Settings.DownloadAudios;
blog.DownloadPhoto = shellService.Settings.DownloadImages;
blog.DownloadVideo = shellService.Settings.DownloadVideos;
blog.DownloadText = shellService.Settings.DownloadTexts;
blog.DownloadQuote = shellService.Settings.DownloadQuotes;
blog.DownloadConversation = shellService.Settings.DownloadConversations;
blog.DownloadLink = shellService.Settings.DownloadLinks;
blog.CreatePhotoMeta = shellService.Settings.CreateImageMeta;
blog.CreateVideoMeta = shellService.Settings.CreateVideoMeta;
blog.CreateAudioMeta = shellService.Settings.CreateAudioMeta;
blog.SkipGif = shellService.Settings.SkipGif;
blog.ForceSize = shellService.Settings.ForceSize;
blog.CheckDirectoryForFiles = shellService.Settings.CheckDirectoryForFiles;
blog.DownloadUrlList = shellService.Settings.DownloadUrlList;
}
public async Task AddBlogAsync(string blogUrl)
{
if (String.IsNullOrEmpty(blogUrl))
{
blogUrl = crawlerService.NewBlogUrl;
}
string blogName = ExtractBlogname(blogUrl);
if (selectionService.BlogFiles.Select(blogs => blogs.Name).ToList().Contains(blogName))
{
shellService.ShowError(null, Resources.BlogAlreadyExist, blogName);
return;
}
var blogPath = shellService.Settings.DownloadLocation;
TumblrBlog blog = new TumblrBlog(ExtractUrl(blogUrl));
blog.Name = blogName;
if (Application.Current.Dispatcher.CheckAccess())
{
selectionService.BlogFiles.Add(blog);
}
else
{
await Application.Current.Dispatcher.BeginInvoke(
DispatcherPriority.Background,
new Action(() =>
{
selectionService.BlogFiles.Add(blog);
}));
}
//var tuple = GetImageUrls(blog);
//blog.TotalCount = tuple.Item1;
blog = await GetMetaInformation(blog);
blog.Online = await IsBlogOnline(blog.Name);
SaveBlog(blog);
}
private void SaveChanges(TumblrBlog blogFile)
{
if (blogFile == null)
{
return;
}
IReadOnlyCollection <IBlog> filesToSave;
if (blogsToSave.Any())
{
filesToSave = managerService.BlogFiles.Where(blogs => blogsToSave.Contains(blogs)).ToArray();
}
else
{
filesToSave = new[] { blogFile };
}
if (blogFile.Dirty)
{
foreach (TumblrBlog blog in filesToSave)
{
blog.DownloadAudio = blogFile.DownloadAudio;
blog.DownloadConversation = blogFile.DownloadConversation;
blog.DownloadLink = blogFile.DownloadLink;
blog.DownloadPhoto = blogFile.DownloadPhoto;
blog.DownloadQuote = blogFile.DownloadQuote;
blog.DownloadText = blogFile.DownloadText;
blog.DownloadVideo = blogFile.DownloadVideo;
blog.CreatePhotoMeta = blogFile.CreatePhotoMeta;
blog.CreateVideoMeta = blogFile.CreateVideoMeta;
blog.CreateAudioMeta = blogFile.CreateAudioMeta;
blog.SkipGif = blogFile.SkipGif;
blog.ForceSize = blogFile.ForceSize;
blog.ForceRescan = blogFile.ForceRescan;
blog.CheckDirectoryForFiles = blogFile.CheckDirectoryForFiles;
blog.DownloadUrlList = blogFile.DownloadUrlList;
blog.Dirty = true;
}
}
RemoveBlogFilesToSave(filesToSave);
}
/// <summary>
///
/// </summary>
/// <param name="blog"></param>
/// <param name="url"></param>
/// <param name="apiMode"></param>
/// <param name="offset"></param>
/// <param name="limit"></param>
public TumblrStatsManager(TumblrBlog blog = null, string url = null, TumblrApiVersion apiMode = TumblrApiVersion.V2Json, int offset = 0, int limit = 0)
{
DocumentManager = new DocumentManager();
ApiVersion = apiMode;
DocumentManager.ApiVersion = apiMode;
TotalPostsPerDocument = (int)NumberOfPostsPerApiDocument.ApiV2;
Blog = blog ?? new TumblrBlog(url);
Blog.Posts = new HashSet <TumblrPost>();
TumblrUrl = WebHelper.RemoveTrailingBackslash(Blog.Url);
TumblrDomain = WebHelper.GetDomainName(TumblrUrl);
ApiQueryLimit = limit;
ApiQueryOffset = offset;
TotalPostsPerDocument = (int)NumberOfPostsPerApiDocument.ApiV2; //20 for JSON, 50 for XML
// Get Blog Info
DocumentManager.GetRemoteBlogInfo(TumblrApiHelper.GeneratePostTypeQueryUrl(TumblrDomain, TumblrPostType.All, 0, 1), Blog);
}
/// <summary>
/// Returns count of all Posts
/// </summary>
/// <returns>A Count of all Posts.</returns>
private static int GetPostCount()
{
string json = string.Empty;
var url = @"http://api.tumblr.com/v2/blog/deliciousanimefeet.tumblr.com/info";
url += "?api_key=uUXKMGxY2yGFCqey98rT9T0jU4ZBke2EgiqPPRhv2eCNIYeuki";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Headers.Add("api_key", "uUXKMGxY2yGFCqey98rT9T0jU4ZBke2EgiqPPRhv2eCNIYeuki");
request.AutomaticDecompression = DecompressionMethods.GZip;
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
using (Stream stream = response.GetResponseStream())
using (StreamReader reader = new StreamReader(stream))
{
json = reader.ReadToEnd();
}
TumblrBlog blog = TumblrBlog.FromJson(json);
return(Convert.ToInt32(blog.Response.Blog.Posts));
}
private Task <TumblrBlog> GetMetaInformation(TumblrBlog blog)
{
return(Task <TumblrBlog> .Factory.StartNew(() =>
{
string url = GetApiUrl(blog.Name, 1);
string authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
var blogDoc = RequestData(url, authHeader);
if (blogDoc != null)
{
blog.Title = blogDoc.response.blog.title;
blog.Description = blogDoc.response.blog.description;
blog.TotalCount = (uint)blogDoc.response.blog.total_posts;
return blog;
}
else
{
return blog;
}
},
TaskCreationOptions.LongRunning));
}
private bool Download(TumblrBlog blog, string fileLocation, string url)
{
Monitor.Enter(blog);
if (blog.Links.Contains(url))
{
Monitor.Exit(blog);
return(false);
}
else
{
Monitor.Exit(blog);
try
{
using (var stream = ThrottledStream.ReadFromURLIntoStream(url, (shellService.Settings.Bandwidth / shellService.Settings.ParallelImages), shellService.Settings.TimeOut))
ThrottledStream.SaveStreamToDisk(stream, fileLocation);
return(true);
}
catch (Exception)
{
return(false);
}
}
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="blog"></param>
/// <param name="saveLocation"></param>
/// <param name="generateLog"></param>
/// <param name="parseSets"></param>
/// <param name="parseJpeg"></param>
/// <param name="parsePng"></param>
/// <param name="parseGif"></param>
/// <param name="imageSize"></param>
/// <param name="offset"></param>
/// <param name="limit"></param>
/// <param name="apiVersion"></param>
public PhotoPostParseManager(TumblrBlog blog = null, string saveLocation = null, bool generateLog = false, bool parseSets = true,
bool parseJpeg = true, bool parsePng = true, bool parseGif = true, ImageSize imageSize = ImageSize.None, int offset = 0, int limit = 0, TumblrApiVersion apiVersion = TumblrApiVersion.V2Json)
{
if (blog != null)
{
TumblrUrl = WebHelper.RemoveTrailingBackslash(blog.Url);
TumblrDomain = WebHelper.GetDomainName(blog.Url);
}
GenerateLog = generateLog;
ApiQueryOffset = offset;
SaveLocation = saveLocation;
ApiQueryPostLimit = limit;
ErrorList = new HashSet <string>();
Blog = blog;
ProcessingStatusCode = ProcessingCode.Ok;
ParsePhotoSets = parseSets;
ParseJpeg = parseJpeg;
ParsePng = parsePng;
ParseGif = parseGif;
ImageSize = imageSize;
ApiVersion = apiVersion;
Blog?.Posts?.Clear();
TotalNumberOfPosts = 0;
ImageList = new HashSet <PhotoPostImage>();
TotalNumberOfImages = 0;
DocumentManager = new RemoteDocumentManager()
{
ApiVersion = apiVersion
};
ImageCommentsList = new Dictionary <string, string>();
}
public void GenerateTagListTest()
{
string logFileName = @"D:\Tumblr\Blogs\jai-envie-de-toi\jai-envie-detoi.log.txt";
SaveFile log = JsonHelper.ReadObjectFromFile <SaveFile>(logFileName);
TumblrBlog blog = log.Blog;
HashSet <string> tags = new HashSet <string>();
SortedSet <string> invalidtags = new SortedSet <string>();
foreach (TumblrPost post in blog.Posts)
{
tags.UnionWith(post.Tags);
foreach (string tag in post.Tags)
{
if (tag.StartsWith(tag.Substring(0, 1).ToUpper()))
{
invalidtags.Add("<a href = '" + post.ShortUrl + "' target='_blank'>" + post.ShortUrl + " : " + tag
+ "</a><br>");
}
}
}
SortedSet <string> tagsSorted = new SortedSet <string>(tags);
//foreach(string tag in tagsSorted)
//{
// if (tag.StartsWith(tag.Substring(0,1).ToUpper()))
// {
// invalidtags.Add(tag);
// }
//}
File.WriteAllText(@"D:\Tumblr\Blogs\jai-envie-de-toi\tags.txt", string.Join(",", tagsSorted));
File.WriteAllLines(@"D:\Tumblr\Blogs\jai-envie-de-toi\invalidtags.html", invalidtags);
}
/// <summary>
///
/// </summary>
/// <param name="fileName"></param>
/// <param name="blog"></param>
public SaveFile(string fileName = "", TumblrBlog blog = null)
{
Filename = fileName;
Blog = blog.Clone();
AddDate();
}
private async Task SaveChangesAsync(TumblrBlog blogFile)
{
if (blogFile == null)
{
return;
}
IReadOnlyCollection <IBlog> filesToSave;
if (blogsToSave.Any())
{
filesToSave = selectionService.BlogFiles.Where(blogs => blogsToSave.Contains(blogs)).ToArray();
}
else
{
filesToSave = new[] { blogFile };
}
if (blogFile.Dirty)
{
//var tasks = Task.Run(() => Parallel.ForEach(filesToSave, blog =>
//{
// blog.DownloadAudio = blogFile.DownloadAudio;
// blog.DownloadConversation = blogFile.DownloadConversation;
// blog.DownloadLink = blogFile.DownloadLink;
// blog.DownloadPhoto = blogFile.DownloadPhoto;
// blog.DownloadQuote = blogFile.DownloadQuote;
// blog.DownloadText = blogFile.DownloadText;
// blog.DownloadVideo = blogFile.DownloadVideo;
// blog.SkipGif = blogFile.SkipGif;
// blog.Dirty = true;
//}));
foreach (TumblrBlog blog in filesToSave)
{
blog.DownloadAudio = blogFile.DownloadAudio;
blog.DownloadConversation = blogFile.DownloadConversation;
blog.DownloadLink = blogFile.DownloadLink;
blog.DownloadPhoto = blogFile.DownloadPhoto;
blog.DownloadQuote = blogFile.DownloadQuote;
blog.DownloadText = blogFile.DownloadText;
blog.DownloadVideo = blogFile.DownloadVideo;
blog.CreatePhotoMeta = blogFile.CreatePhotoMeta;
blog.CreateVideoMeta = blogFile.CreateVideoMeta;
blog.CreateAudioMeta = blogFile.CreateAudioMeta;
blog.SkipGif = blogFile.SkipGif;
blog.Dirty = true;
}
}
//try
//{
// await Task.WhenAll(tasks);
//}
//catch (Exception ex)
//{
// Logger.Error("SaveChangesAsync: {0}", ex);
// if (filesToSave.Count() == 1)
// {
// shellService.ShowError(ex, Resources.CouldNotSaveBlog, filesToSave.First().Name);
// }
// else
// {
// shellService.ShowError(ex, Resources.CouldNotSaveBlog);
// }
//}
//finally
//{
RemoveBlogFilesToSave(filesToSave);
//}
}
private IReadOnlyList <IBlog> GetIBlogsCore(string directory)
{
Logger.Verbose("ManagerController:GetIBlogsCore Start");
var blogs = new List <IBlog>();
var failedToLoadBlogs = new List <string>();
string[] supportedFileTypes = Enum.GetNames(typeof(BlogTypes)).ToArray();
int[] validCollectionIds = _shellService.Settings.Collections.Select(s => s.Id).ToArray();
foreach (string filename in Directory.GetFiles(directory, "*").Where(
fileName => supportedFileTypes.Any(fileName.Contains) &&
!fileName.Contains("_files")))
{
//TODO: Refactor
try
{
IBlog blog = null;
if (filename.EndsWith(BlogTypes.tumblr.ToString()))
{
blog = new TumblrBlog().Load(filename);
}
if (filename.EndsWith(BlogTypes.tmblrpriv.ToString()))
{
blog = new TumblrHiddenBlog().Load(filename);
}
if (filename.EndsWith(BlogTypes.tlb.ToString()))
{
blog = new TumblrLikedByBlog().Load(filename);
}
if (filename.EndsWith(BlogTypes.tumblrsearch.ToString()))
{
blog = new TumblrSearchBlog().Load(filename);
}
if (filename.EndsWith(BlogTypes.tumblrtagsearch.ToString()))
{
blog = new TumblrTagSearchBlog().Load(filename);
}
if (filename.EndsWith(BlogTypes.twitter.ToString()))
{
blog = new TwitterBlog().Load(filename);
}
if (blog != null)
{
if (!validCollectionIds.Contains(blog.CollectionId))
{
blog.CollectionId = 0;
}
blogs.Add(blog);
}
}
catch (SerializationException ex)
{
failedToLoadBlogs.Add(ex.Data["Filename"].ToString());
}
}
if (failedToLoadBlogs.Any())
{
string failedBlogNames = failedToLoadBlogs.Aggregate((a, b) => a + ", " + b);
Logger.Verbose("ManagerController:GetIBlogsCore: {0}", failedBlogNames);
_shellService.ShowError(new SerializationException(), Resources.CouldNotLoadLibrary, failedBlogNames);
}
Logger.Verbose("ManagerController.GetIBlogsCore End");
return(blogs);
}
public Tuple <uint, List <string> > GetImageUrls(TumblrBlog blog, IProgress <DataModels.DownloadProgress> progress, CancellationToken ct, PauseToken pt)
{
int totalPosts = 0;
int numberOfPostsCrawled = 0;
uint totalImages;
List <string> images = new List <string>();
string url = GetApiUrl(blog.Name, 1);
string authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
var blogDoc = RequestData(url, authHeader);
totalPosts = blogDoc.response.blog.total_posts;
// Generate URL list of Images
// the api v2 shows 20 posts at max, determine the number of pages to crawl
int totalPages = (totalPosts / 20) + 1;
Parallel.For(0, totalPages,
new ParallelOptions {
MaxDegreeOfParallelism = (shellService.Settings.ParallelImages / selectionService.ActiveItems.Count)
},
(i, state) =>
{
if (ct.IsCancellationRequested)
{
state.Break();
}
if (pt.IsPaused)
{
pt.WaitWhilePausedWithResponseAsyc().Wait();
}
try
{
// check for tags -- crawling for all images here
if (blog.Tags == null || blog.Tags.Count() == 0)
{
DataModels.TumblrJson document = null;
// get 20 posts per crawl/page
url = GetApiUrl(blog.Name, 20, i * 20);
authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
document = RequestData(url, authHeader);
if (shellService.Settings.DownloadImages == true)
{
foreach (Datamodels.Post post in document.response.posts.Where(posts => posts.type.Equals("photo")))
{
foreach (DataModels.Photo photo in post.photos)
{
var imageUrl = photo.alt_sizes.ElementAt(shellService.Settings.ImageSizes.IndexOf(shellService.Settings.ImageSize.ToString())).url;
if (shellService.Settings.SkipGif == true && imageUrl.EndsWith(".gif"))
{
continue;
}
Monitor.Enter(images);
images.Add(imageUrl);
Monitor.Exit(images);
}
}
}
if (shellService.Settings.DownloadVideos == true)
{
foreach (DataModels.Post post in document.response.posts.Where(posts => posts.type.Equals("video")))
{
if (shellService.Settings.VideoSize == 1080)
{
Monitor.Enter(images);
images.Add(post.video_url);
Monitor.Exit(images);
}
if (shellService.Settings.VideoSize == 480)
{
Monitor.Enter(images);
images.Add(post.video_url.Insert(post.video_url.LastIndexOf("."), "_480"));
Monitor.Exit(images);
}
}
}
}
// crawling only for tagged images
else
{
List <string> tags = blog.Tags.Split(',').Select(x => x.Trim()).ToList();
DataModels.TumblrJson document = null;
// get 20 posts per crawl/page
url = GetApiUrl(blog.Name, 20, i * 20);
authHeader = shellService.OAuthManager.GenerateauthHeader(url, "GET");
document = RequestData(url, authHeader);
if (shellService.Settings.DownloadImages == true)
{
foreach (Datamodels.Post post in document.response.posts.Where(posts => posts.tags.Any(tag => tags.Equals(tag)) && posts.type.Equals("photo")))
{
foreach (DataModels.Photo photo in post.photos ?? new List <Datamodels.Photo>())
{
var imageUrl = photo.alt_sizes.ElementAt(shellService.Settings.ImageSizes.IndexOf(shellService.Settings.ImageSize.ToString())).url;
if (shellService.Settings.SkipGif == true && imageUrl.EndsWith(".gif"))
{
continue;
}
Monitor.Enter(images);
images.Add(imageUrl);
Monitor.Exit(images);
}
}
}
if (shellService.Settings.DownloadVideos == true)
{
foreach (DataModels.Post post in document.response.posts.Where(posts => posts.tags.Any(tag => tags.Equals(tag)) && posts.type.Equals("video")))
{
if (shellService.Settings.VideoSize == 1080)
{
Monitor.Enter(images);
images.Add(post.video_url);
Monitor.Exit(images);
}
if (shellService.Settings.VideoSize == 480)
{
Monitor.Enter(images);
images.Add(post.video_url.Insert(post.video_url.LastIndexOf("."), "_480"));
Monitor.Exit(images);
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Data);
}
numberOfPostsCrawled += 20;
var newProgress = new DataModels.DownloadProgress();
newProgress.Progress = string.Format(CultureInfo.CurrentCulture, Resources.ProgressGetUrl, numberOfPostsCrawled, totalPosts);
progress.Report(newProgress);
}
);
images = images.Distinct().ToList();
totalImages = (uint)images.Count;
return(Tuple.Create(totalImages, images));
}
