public ICrawler GetCrawler(IBlog blog, IProgress <DownloadProgress> progress, PauseToken pt, CancellationToken ct)
{
blog.DownloadedItemsNew = 0;
IPostQueue <AbstractPost> postQueue = GetProducerConsumerCollection();
IFiles files = LoadFiles(blog);
IWebRequestFactory webRequestFactory = GetWebRequestFactory();
IImgurParser imgurParser = GetImgurParser(webRequestFactory, ct);
IGfycatParser gfycatParser = GetGfycatParser(webRequestFactory, ct);
switch (blog.BlogType)
{
case BlogTypes.tumblr:
IPostQueue <CrawlerData <Post> > jsonApiQueue = GetJsonQueue <Post>();
return(new TumblrBlogCrawler(shellService, crawlerService, webRequestFactory, cookieService,
GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetJsonDownloader(jsonApiQueue, blog, pt, ct),
GetTumblrApiJsonToTextParser(blog), GetTumblrParser(), imgurParser, gfycatParser, GetWebmshareParser(),
GetUguuParser(), GetCatBoxParser(), postQueue,
jsonApiQueue, blog, progress, pt, ct));
case BlogTypes.tmblrpriv:
IPostQueue <CrawlerData <DataModels.TumblrSvcJson.Post> > jsonSvcQueue =
GetJsonQueue <DataModels.TumblrSvcJson.Post>();
return(new TumblrHiddenCrawler(shellService, crawlerService, webRequestFactory,
cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct),
GetJsonDownloader(jsonSvcQueue, blog, pt, ct), GetTumblrSvcJsonToTextParser(blog), GetTumblrParser(),
imgurParser, gfycatParser, GetWebmshareParser(), GetUguuParser(),
GetCatBoxParser(), postQueue, jsonSvcQueue, blog, progress, pt, ct));
case BlogTypes.tlb:
return(new TumblrLikedByCrawler(shellService, crawlerService, webRequestFactory,
cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetTumblrParser(),
imgurParser, gfycatParser, GetWebmshareParser(), GetUguuParser(),
GetCatBoxParser(), postQueue, blog, progress, pt, ct));
case BlogTypes.tumblrsearch:
IPostQueue <CrawlerData <string> > jsonQueue = GetJsonQueue <string>();
return(new TumblrSearchCrawler(shellService, crawlerService, webRequestFactory,
cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetJsonDownloader(jsonQueue, blog, pt, ct),
GetTumblrApiJsonToTextParser(blog), GetTumblrParser(), imgurParser, gfycatParser, GetWebmshareParser(),
GetUguuParser(), GetCatBoxParser(), postQueue, jsonQueue, blog, progress, pt, ct));
case BlogTypes.tumblrtagsearch:
IPostQueue <CrawlerData <DataModels.TumblrTaggedSearchJson.Datum> > jsonTagSearchQueue =
GetJsonQueue <DataModels.TumblrTaggedSearchJson.Datum>();
return(new TumblrTagSearchCrawler(shellService, crawlerService, webRequestFactory,
cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct),
GetJsonDownloader(jsonTagSearchQueue, blog, pt, ct), GetTumblrParser(),
imgurParser, gfycatParser, GetWebmshareParser(), GetUguuParser(),
GetCatBoxParser(), postQueue, jsonTagSearchQueue, blog, progress, pt, ct));
case BlogTypes.twitter:
IPostQueue <CrawlerData <Tweet> > jsonTwitterQueue = GetJsonQueue <Tweet>();
return(new TwitterCrawler(shellService, crawlerService, progress, webRequestFactory,
cookieService, postQueue, jsonTwitterQueue, blog, GetTwitterDownloader(progress, blog, files, postQueue, pt, ct),
GetJsonDownloader(jsonTwitterQueue, blog, pt, ct), pt, ct));
default:
throw new ArgumentException("Website is not supported!", nameof(blog));
}
}
public ICrawler GetCrawler(IBlog blog, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress, IShellService shellService, ICrawlerService crawlerService, IManagerService managerService)
{
IPostQueue <TumblrPost> postQueue = GetProducerConsumerCollection();
IFiles files = LoadFiles(blog, managerService);
IWebRequestFactory webRequestFactory = GetWebRequestFactory();
IImgurParser imgurParser = GetImgurParser(webRequestFactory, ct);
IGfycatParser gfycatParser = GetGfycatParser(webRequestFactory, ct);
switch (blog.BlogType)
{
case BlogTypes.tumblr:
IPostQueue <TumblrCrawlerData <DataModels.TumblrApiJson.Post> > jsonApiQueue = GetJsonQueue <DataModels.TumblrApiJson.Post>();
return(new TumblrBlogCrawler(shellService, ct, pt, progress, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(ct, pt, progress, shellService, crawlerService, managerService, blog, files, postQueue), GetTumblrJsonDownloader(shellService, ct, pt, jsonApiQueue, crawlerService, blog), GetTumblrApiJsonToTextParser(blog), imgurParser, gfycatParser, GetWebmshareParser(), postQueue, jsonApiQueue, blog));
case BlogTypes.tmblrpriv:
IPostQueue <TumblrCrawlerData <DataModels.TumblrSvcJson.Post> > jsonSvcQueue = GetJsonQueue <DataModels.TumblrSvcJson.Post>();
return(new TumblrHiddenCrawler(shellService, ct, pt, progress, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(ct, pt, progress, shellService, crawlerService, managerService, blog, files, postQueue), GetTumblrJsonDownloader(shellService, ct, pt, jsonSvcQueue, crawlerService, blog), GetTumblrSvcJsonToTextParser(blog), imgurParser, gfycatParser, GetWebmshareParser(), postQueue, jsonSvcQueue, blog));
case BlogTypes.tlb:
return(new TumblrLikedByCrawler(shellService, ct, pt, progress, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(ct, pt, progress, shellService, crawlerService, managerService, blog, files, postQueue), postQueue, blog));
case BlogTypes.tumblrsearch:
return(new TumblrSearchCrawler(shellService, ct, pt, progress, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(ct, pt, progress, shellService, crawlerService, managerService, blog, files, postQueue), postQueue, blog));
case BlogTypes.tumblrtagsearch:
return(new TumblrTagSearchCrawler(shellService, ct, pt, progress, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(ct, pt, progress, shellService, crawlerService, managerService, blog, files, postQueue), postQueue, blog));
default:
throw new ArgumentException("Website is not supported!", "blogType");
}
}
public TumblrBlogCrawler(IShellService shellService, CancellationToken ct, PauseToken pt,
IProgress <DownloadProgress> progress, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, IDownloader downloader, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, BlockingCollection <TumblrPost> producerConsumerCollection, IBlog blog)
: base(shellService, ct, pt, progress, crawlerService, webRequestFactory, cookieService, downloader, producerConsumerCollection, blog)
{
this.imgurParser = imgurParser;
this.gfycatParser = gfycatParser;
this.webmshareParser = webmshareParser;
}
protected AbstractTumblrCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService,
ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IUguuParser uguuParser,
ICatBoxParser catboxParser, IPostQueue <AbstractPost> postQueue, IBlog blog, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader,
IProgress <DownloadProgress> progress, PauseToken pt, CancellationToken ct)
: base(shellService, crawlerService, progress, webRequestFactory, cookieService, postQueue, blog, downloader, pt, ct)
{
this.crawlerDataDownloader = crawlerDataDownloader;
this.crawlerDataDownloader?.ChangeCancellationToken(Ct);
this.TumblrParser = tumblrParser;
this.ImgurParser = imgurParser;
this.GfycatParser = gfycatParser;
this.WebmshareParser = webmshareParser;
this.UguuParser = uguuParser;
this.CatboxParser = catboxParser;
}
protected AbstractTumblrCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService,
ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser,
IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser,
ICatBoxParser catboxParser, IPostQueue <TumblrPost> postQueue, IBlog blog, IProgress <DownloadProgress> progress, PauseToken pt, CancellationToken ct)
: base(shellService, crawlerService, progress, webRequestFactory, cookieService, postQueue, blog, pt, ct)
{
this.TumblrParser = tumblrParser;
this.ImgurParser = imgurParser;
this.GfycatParser = gfycatParser;
this.WebmshareParser = webmshareParser;
this.MixtapeParser = mixtapeParser;
this.UguuParser = uguuParser;
this.SafemoeParser = safemoeParser;
this.LolisafeParser = lolisafeParser;
this.CatboxParser = catboxParser;
}
public TumblrHiddenCrawler(IShellService shellService, CancellationToken ct, PauseToken pt, IProgress <DownloadProgress> progress,
ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, IDownloader downloader,
ICrawlerDataDownloader crawlerDataDownloader, ITumblrToTextParser <Post> tumblrJsonParser, IImgurParser imgurParser,
IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IPostQueue <TumblrPost> postQueue,
IPostQueue <TumblrCrawlerData <Post> > jsonQueue, IBlog blog)
: base(shellService, ct, progress, webRequestFactory, cookieService, postQueue, blog)
{
this.crawlerService = crawlerService;
this.downloader = downloader;
this.pt = pt;
this.tumblrJsonParser = tumblrJsonParser;
this.imgurParser = imgurParser;
this.gfycatParser = gfycatParser;
this.webmshareParser = webmshareParser;
this.jsonQueue = jsonQueue;
this.crawlerDataDownloader = crawlerDataDownloader;
}
public TumblrTagSearchCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory,
ISharedCookieService cookieService, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader, ITumblrParser tumblrParser, IImgurParser imgurParser,
IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser,
ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser,
IPostQueue <TumblrPost> postQueue, IPostQueue <TumblrCrawlerData <Datum> > jsonQueue, IBlog blog, IProgress <DownloadProgress> progress, PauseToken pt, CancellationToken ct)
: base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser,
webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog, downloader, progress, pt, ct)
{
this.downloader = downloader;
this.jsonQueue = jsonQueue;
this.crawlerDataDownloader = crawlerDataDownloader;
}
public TumblrLikedByCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory,
ISharedCookieService cookieService, IDownloader downloader, ITumblrParser tumblrParser, IImgurParser imgurParser,
IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IUguuParser uguuParser, ICatBoxParser catboxParser,
IPostQueue <AbstractPost> postQueue, IBlog blog, IProgress <DownloadProgress> progress, PauseToken pt, CancellationToken ct)
: base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser,
webmshareParser, uguuParser, catboxParser, postQueue, blog, downloader, null,
progress, pt, ct)
{
this.downloader = downloader;
}
public TumblrHiddenCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory,
ISharedCookieService cookieService, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader,
ITumblrToTextParser <Post> tumblrJsonParser, ITumblrParser tumblrParser, IImgurParser imgurParser,
IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IUguuParser uguuParser, ICatBoxParser catboxParser,
IPostQueue <AbstractPost> postQueue, IPostQueue <CrawlerData <Post> > jsonQueue, IBlog blog, IProgress <DownloadProgress> progress, PauseToken pt, CancellationToken ct)
: base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser,
webmshareParser, uguuParser, catboxParser, postQueue, blog, downloader, crawlerDataDownloader, progress, pt,
ct)
{
this.downloader = downloader;
this.tumblrJsonParser = tumblrJsonParser;
this.jsonQueue = jsonQueue;
}
