public Task GetRssFeed()
{
_logger.Trace("Starting scraping job");
using (NewsFeedScraper scraper = new NewsFeedScraper())
{
try
{
scraper.GoToDashboard(_options.Value);
Task.Delay(TimeSpan.FromSeconds(10)).Wait();
Feed feed = scraper.TryScrapeFeed(_options.Value);
if (feed != null && feed.Items != null)
{
foreach (var item in feed.Items)
{
Console.WriteLine($"{item.Id} - {item.JobString} - {item.Url} - {item.Description}");
}
}
}
catch (Exception e)
{
_logger.Error(e);
scraper.SaveSource();
}
}
return(Task.FromResult(true));
}
public void Execute(IJobExecutionContext context)
{
_logger.Trace("Starting scraping job");
ScraperConfig config = new ScraperConfig
{
UserName = _configuration.AppSettings["UserName"],
Password = _configuration.AppSettings["Password"],
Url = _configuration.AppSettings["Url"],
LoginUrl = _configuration.AppSettings["LoginUrl"],
ApiUrl = _configuration.AppSettings["FeedUrl"],
Proxies = new [] { "91.142.84.182:3128" } //TODO File.ReadAllLines(@"Data\fastproxies.txt")
};
using (NewsFeedScraper scraper = _scraperFactory.Create(config))
{
try
{
scraper.GoToDashboard(config);
Task.Delay(TimeSpan.FromSeconds(10)).Wait();
Feed feed = scraper.TryScrapeFeed(config);
if (feed != null && feed.Items != null)
{
foreach (var item in feed.Items)
{
_logger.Trace($"{item.Id} - {item.JobString} - {item.Title} - {item.Url} - {item.Description}");
}
_dataService.UpdateFeedAsync(feed).Wait();
}
}
catch (Exception e)
{
_logger.Error(e);
// take screenshot
scraper.TakeScreenshot();
scraper.SaveSource();
}
}
}
public void Scrape_AnyState_ReturnsFeed()
{
ScraperConfig config = new ScraperConfig
{
UserName = _configuration.AppSettings["UserName"],
Password = _configuration.AppSettings["Password"],
Url = "https://www.freelancer.com/dashboard",
ApiUrl = "https://www.freelancer.com/ajax/notify/live-feed/pre-populated.php"
};
bool result = true;
using (NewsFeedScraper scraper = new NewsFeedScraper(null))
{
try
{
scraper.GoToDashboard(config);
Feed feed = scraper.TryScrapeFeed(config);
if (feed != null && feed.Items != null)
{
foreach (var item in feed.Items)
{
_logger.Trace($"{item.Id} - {item.JobString} - {item.Title} - {item.Url} - {item.Description}");
}
}
}
catch (Exception e)
{
_logger.Error(e);
// take screenshot
scraper.TakeScreenshot();
scraper.SaveSource();
result = false;
}
}
result.Should().BeTrue();
}
public void Scrape_AnyState_ReturnsFeed()
{
var configOptions = new ConfigOptions
{
UserName = "",
Password = "",
Url = "https://www.freelancer.com/dashboard",
FeedUrl = "https://www.freelancer.com/ajax/notify/live-feed/pre-populated.php"
};
bool result = true;
using (NewsFeedScraper scraper = new NewsFeedScraper())
{
try
{
scraper.GoToDashboard(configOptions);
Feed feed = scraper.TryScrapeFeed(configOptions);
if (feed != null && feed.Items != null)
{
foreach (var item in feed.Items)
{
_logger.Trace($"{item.Id} - {item.JobString} - {item.Url} - {item.Description}");
}
}
}
catch (Exception e)
{
_logger.Error(e);
scraper.SaveSource();
result = false;
}
}
result.Should().BeTrue();
}