public static async Task<IPicture> ScrapeAsync(IScraper scraper)
{
using (var client = new HttpClient())
{
// It looks like dilbert.com mucks with the charset ... intentionally?
//var html = await client.GetStringAsync(scraper.Url);
HttpResponseMessage response = await client.GetAsync(scraper.Url);
response.EnsureSuccessStatusCode();
string charset = response.Content.Headers.ContentType.CharSet;
if (charset.EndsWith("lias"))
{
charset = charset.Replace("lias", String.Empty);
response.Content.Headers.ContentType.CharSet = charset;
}
string html = await response.Content.ReadAsStringAsync();
Uri imageUrl = await scraper.GetImageUrlAsync(html);
// be sure the URL is really an image
//Image image = await GetImageAsync(imageUrl);
//byte[] imageBytes = await client.GetByteArrayAsync(imageUrl);
IImage image = await GetImageAsync(imageUrl);
return new Picture(scraper, imageUrl, image);
}
}
public GameResultsScrapedEventArgs(ICoreService eventCoreInstance, string gameFileName, IList<IGameScrapeResult> scrapedResults, IScraper scraper)
: base(eventCoreInstance)
{
this.GameFileName = gameFileName;
this.ScrapedResults = scrapedResults;
this.GameScraper = scraper;
}
public DownloadViewModel(DownloadedChapterInfo downloadInfo, ISemaphore downloadSemaphore)
{
if (downloadInfo == null)
throw new ArgumentNullException("downloadInfo");
if (downloadSemaphore == null)
throw new ArgumentNullException("downloadSemaphore");
if (downloadInfo.ChapterRecord == null)
throw new ArgumentException("Chapter record is invalid.", "downloadInfo");
if (String.IsNullOrEmpty(downloadInfo.ChapterRecord.ChapterId))
throw new ArgumentException("Chapter record id is invalid.", "downloadInfo");
if (downloadInfo.ChapterRecord.MangaRecord == null)
throw new ArgumentException("Manga record is invalid.", "downloadInfo");
if (String.IsNullOrEmpty(downloadInfo.ChapterRecord.MangaRecord.MangaId))
throw new ArgumentException("Manga record id is invalid.", "downloadInfo");
_downloadInfo = downloadInfo;
_downloadSemaphore = downloadSemaphore;
_scraper = ScraperLoader.Instance.AllScrapers.FirstOrDefault(s => s.ScraperGuid == downloadInfo.ChapterRecord.Scraper);
if (_scraper != null)
{
_downloader = _scraper.GetDownloader();
// register downloader events
_downloader.DownloadProgress += _downloader_DownloadProgress;
_downloader.DownloadCompleted += _downloader_DownloadCompleted;
}
if (!String.IsNullOrEmpty(_downloadInfo.Path))
{
// file was already downloaded
State = DownloadState.Unknown;
Completed = true;
}
else
{
// we will be downloading the file now
State = DownloadState.Ok;
Completed = false;
}
CurrentActionText = String.Empty;
_cancelDownloadCommand = new RelayCommand(Cancel, x => !Completed);
_removeDownloadCommand = new RelayCommand(Remove);
_openDownloadCommand = new RelayCommand(Open, x => DownloadExists);
_retryDownloadCommand = new RelayCommand(RetryDownload, x => _downloader != null
&& Completed
&& !DownloadExists
&& !String.IsNullOrEmpty(_downloadInfo.DownloadFolder));
CancelText = ButtonCancelText;
}
public RecentMangaRecord(IMangaRecord mangaRecord)
{
if (mangaRecord == null)
throw new ArgumentNullException("mangaRecord");
_mangaRecord = mangaRecord;
_scraper = ScraperLoader.Instance.AllScrapers.FirstOrDefault(s => s.ScraperGuid == _mangaRecord.Scraper);
}
public Reports()
{
_scraper = App.Container.Resolve<IScraper>();
InitializeComponent();
var reports = _scraper.FetchAllScrapeReports();
LvReports.ItemsSource = reports;
}
/**/
public class List1
{
public string Tag { get; set; }
public byte[] Resim1 { get; set; }
public byte[] Resim2 { get; set; }
}
public class List2
{
public string Tag { get; set; }
public byte[] Resim { get; set; }
}
/**/
public MainWindow()
{
_scraper = App.Container.Resolve<IScraper>();
_scraper.ScrapeStarted += ScraperOnScrapeStarted;
_scraper.ScrapeEnded += ScraperOnScrapeEnded;
_scraper.ResourceScraped += ScraperOnResourceScraped;
InitializeComponent();
}
public DrNuClient(IResourceUriScraper resourceUriScraper, IResourceScraper resourceScraper, IResourceMapper resourceMapper, IProgramSlugScraper programSlugScraper, IProgramUriScraper programUriScraper)
{
if (resourceUriScraper == null) throw new ArgumentNullException("resourceUriScraper");
if (resourceScraper == null) throw new ArgumentNullException("resourceScraper");
if (resourceMapper == null) throw new ArgumentNullException("resourceMapper");
if (programSlugScraper == null) throw new ArgumentNullException("programSlugScraper");
if (programUriScraper == null) throw new ArgumentNullException("programUriScraper");
_resourceUriScraper = resourceUriScraper;
_programUriScraper = programUriScraper;
_resourceScraper = resourceScraper;
_resourceMapper = resourceMapper;
_programSlugScraper = programSlugScraper;
}
public SearchService(IScraper[] scrapers, IAggregator aggregator)
{
if (scrapers == null)
{
throw new ArgumentNullException("scrapers");
}
if (aggregator == null)
{
throw new ArgumentNullException("aggregator");
}
this._aggregator = aggregator;
this._scrapers = scrapers;
}
public LiveCharts_co_uk_PriceFeed(IScraper scraper)
{
Condition.Requires(scraper).IsNotNull();
this._scraper = scraper;
#region Set starting urls
this._listUrls = new string[]
{
"http://www.livecharts.co.uk/share_map.php?letter=0-9",
"http://www.livecharts.co.uk/share_map.php?letter=a",
"http://www.livecharts.co.uk/share_map.php?letter=b",
"http://www.livecharts.co.uk/share_map.php?letter=c",
"http://www.livecharts.co.uk/share_map.php?letter=d",
"http://www.livecharts.co.uk/share_map.php?letter=e",
"http://www.livecharts.co.uk/share_map.php?letter=f",
"http://www.livecharts.co.uk/share_map.php?letter=g",
"http://www.livecharts.co.uk/share_map.php?letter=h",
"http://www.livecharts.co.uk/share_map.php?letter=i",
"http://www.livecharts.co.uk/share_map.php?letter=j",
"http://www.livecharts.co.uk/share_map.php?letter=k",
"http://www.livecharts.co.uk/share_map.php?letter=l",
"http://www.livecharts.co.uk/share_map.php?letter=m",
"http://www.livecharts.co.uk/share_map.php?letter=n",
"http://www.livecharts.co.uk/share_map.php?letter=o",
"http://www.livecharts.co.uk/share_map.php?letter=p",
"http://www.livecharts.co.uk/share_map.php?letter=q",
"http://www.livecharts.co.uk/share_map.php?letter=r",
"http://www.livecharts.co.uk/share_map.php?letter=s",
"http://www.livecharts.co.uk/share_map.php?letter=t",
"http://www.livecharts.co.uk/share_map.php?letter=u",
"http://www.livecharts.co.uk/share_map.php?letter=v",
"http://www.livecharts.co.uk/share_map.php?letter=w",
"http://www.livecharts.co.uk/share_map.php?letter=x",
"http://www.livecharts.co.uk/share_map.php?letter=y",
"http://www.livecharts.co.uk/share_map.php?letter=z"
}.ToList<string>();
#endregion
}
private void LoadScraperPreview(IScraper scraper)
{
if(!(scraper is IPreview))
return;
_requestQueue.Add(
() => ((IPreview) scraper).Preview(),
(r, e) =>
{
var records = r as IEnumerable<IMangaRecord>;
if (e == null && r != null)
{
lock (_syncRoot)
{
// just replace collection -> this is easier than removing and then adding records
Mangas = new AsyncObservableCollection<IMangaRecord>(records);
OnPropertyChanged(() => Mangas);
}
}
}
);
}
internal Picture(IScraper scraper, Uri url, IImage image)
: this(scraper.Name, scraper.DisplayName, scraper.Rating, scraper.Categories)
{
this.url = url;
this.image = image;
}
private async Task ScrapAsync(IScraper scraper)
{
await Task.Run(() =>
{
var progress = new Progress<double>(x =>
{
DispatcherHelper.CheckBeginInvokeOnUI(() => this.ScrapProgress = x);
});
var tracks = scraper.Scrap(progress)
.Distinct(new TrackComparer());
foreach (var track in tracks)
{
DispatcherHelper.CheckBeginInvokeOnUI(() => this.ScrapedTracks.Add(track));
}
});
}
public ImageScraperPresenter(IScraper view)
{
_view = view;
}
public ScraperRunner(IScraper scraper)
{
if (scraper == null) throw new ArgumentNullException("scraper");
this.scraper = scraper;
highlighter = new Highligher();
}
public CrawlerValidator(IScraper scraper)
{
_scraper = scraper;
}
public CrawlWebRunner(IScraper scraper)
{
_scraper = scraper;
}
public TrackerClient(IAnnouncer[] announcers, IScraper[] scrapers)
{
Announcers = announcers;
Scrapers = scrapers;
}
public GameInfoScrapedEventArgs(ICoreService eventCoreInstance, IGameInfo gameInfo, IScraper gameScraper)
: base(eventCoreInstance, gameInfo)
{
this.GameScraper = gameScraper;
}
public ScraperException(TickerSymbol symbol, IScraper scraper, Exception innerException)
: this(String.Format("Error looking up symbol {0} from {1}.", symbol, scraper.ProviderName), innerException)
{
}
public void TestInitialize()
{
_view = new TestScraperView();
_presenter = new ImageScraperPresenter(_view);
}
public ScraperInfo(IScraper scraper, bool enabled)
{
if (scraper == null)
throw new ArgumentNullException("scraper");
_scraper = scraper;
Enabled = enabled;
}
public TestController()
{
_scrapeHub = GlobalHost.ConnectionManager.GetHubContext<ScrapeHub>();
_scraper = MvcApplication.Container.Resolve<IScraper>();
_log = LogManager.GetCurrentClassLogger();
}
private async Task<List<IManga>> GetMangas(IScraper scraper, string value) {
var mangas = await scraper.Mangas();
return mangas.Where(e => e.Name.ToLowerInvariant().Contains(value.ToLowerInvariant())).ToList();
}
