public ScrapersManager(IInputDataProvider inputDataProvider, IDataHandler <Person> dataHandler, IScrapper <Person> scrapper, ILog log)
{
_inputDataProvider = inputDataProvider;
_dataHandler = dataHandler;
_scrapper = scrapper;
_log = log;
}
public ScrapperModule(IScrapperConfiguration scrapperConfiguration, IScrapper scrapper,
IEnumerable <IParser> parsers) : base("Radio")
{
_scrapperConfiguration = scrapperConfiguration;
_parsers = parsers;
Get["/Get/{radio}"] = parameters =>
{
if (!TryGetRadio((string)parameters.radio, out var radioConfiguration))
{
throw new NotFoundErrorException($"Radio {(string) parameters.radio} not found");
}
var parser = GetParser(radioConfiguration.Name);
var song = scrapper.Scrap(radioConfiguration.Uri, parser);
return(Response.AsJson(song));
};
Get["/GetAsync/{radio}", true] = async(parameters, ct) =>
{
if (!TryGetRadio((string)parameters.radio, out var radioConfiguration))
{
throw new NotFoundErrorException($"Radio {(string)parameters.radio} not found");
}
var parser = GetParser(radioConfiguration.Name);
var song = await scrapper.ScrapAsync(radioConfiguration.Uri, parser);
return(Response.AsJson(song));
};
}
static void Main(string[] args)
{
IScrapper scrapper = null;
string entryUrl = null;
string outputDirectory = null;
Parser.Default.ParseArguments <Options>(args)
.WithParsed(o => {
switch (o.Type)
{
case PodcastType.HowToSaveMoney:
scrapper = new HowToSaveMoney.Scrapper.Scrapper();
entryUrl = "https://jakoszczedzacpieniadze.pl/podcast";
break;
case PodcastType.ForeignDailyNews:
scrapper = new ForeignDailyNews.Scrapper.Scrapper();
entryUrl = "http://www.dzialzagraniczny.pl/category/podcasty/";
break;
default:
throw new System.ArgumentNullException(nameof(Options.Type));
}
outputDirectory = string.IsNullOrEmpty(o.OutputDirectory) ? DefaultOutput : o.OutputDirectory;
});
if (scrapper != null)
{
var downloader = new Downloader(scrapper, outputDirectory, entryUrl);
downloader.Run().GetAwaiter().GetResult();
}
}
public DownloadTask(IScrapper <State, University> universities,
IScrapper <University, Specialty> specialties,
IScrapper <Specialty, Abiturient> abiturients)
{
_universities = universities;
_specialties = specialties;
_abiturients = abiturients;
}
public Downloader(IScrapper scrapper, string outputPath, string entryUrl)
{
_scrapper = scrapper;
_outputPath = outputPath;
_entryUrl = entryUrl;
_articlesCacheFilePath = Path.Combine(_outputPath, "articles.txt");
_downloadUrlsFilePath = Path.Combine(_outputPath, "download.txt");
}
public ScrapperService(
IScrapper scrapper,
IEventRepository eventRepository,
IEventDateRepository eventDateRepository
)
{
_eventRepository = eventRepository;
_eventDateRepository = eventDateRepository;
_scrapper = scrapper;
}
static async Task MainAsync(IScrapper scrapper, ILogger logger)
{
try
{
await scrapper.RetreiveDataAsync();
logger.LogInformation("COMPLETED.");
}
catch (Exception ex)
{
logger.LogCritical(ex, "Unexpected error.Exit.");
Console.ReadKey();
}
}
public ChapterSelector(IScrapper scrapper)
{
_scrapper = scrapper;
InitializeComponent();
// MyListView.ItemsSource = scrapper.SelectedManga.Chapters;
ChapterSelectorViewModel =
new ChapterSelectorViewModel(scrapper.SelectedManga, scrapper.SelectedManga.Chapters)
{
IsFavorite = _scrapper.SelectedManga.IsFavorite
};
BindingContext = ChapterSelectorViewModel;
}
public void AddConfigurations(IScrapper scrapper, ScrapperRunnerConfiguration runnerConfig)
{
if (_scrapperConfigs.ContainsKey(scrapper))
{
_scrapperConfigs[scrapper].Add(runnerConfig);
}
else
{
_scrapperConfigs.Add(scrapper,
new HashSet <ScrapperRunnerConfiguration> {
runnerConfig
});
}
}
static void Main(string[] args)
{
if (args.Count() == 0)
{
Console.WriteLine("pricer location <hemnet|booli> <area>");
Console.WriteLine("pricer get <location_id> <type> <timespan> <filepath> <hemnet|booli>");
return;
}
if (args.Count() == 3)
{
var task = ScrapperFactory.GetLocations(args[1], args[2]);
Console.WriteLine("Waiting result from providers...");
task.Wait();
var locations = task.Result;
Console.WriteLine(String.Join("\n", locations.Select(x => x.GetId() + "\t" + x.GetName() + "\t" + x.GetAreaName()).ToArray()));
}
if (args.Count() == 6)
{
List <string> locationIds = args[1].Split(";").ToList();
List <string> type = args[2].Split(";").ToList();
IScrapper scrapper = ScrapperFactory.GetScrapper(args[5], locationIds, type, args[3]);
var task = scrapper.DoRequest();
Console.WriteLine("Waiting result from providers...");
task.Wait();
var priceInfos = task.Result;
using (StreamWriter sw = new StreamWriter(args[4]))
{
sw.WriteLine("Total Object: " + priceInfos.Count);
Console.WriteLine("Total Object: " + priceInfos.Count);
if (priceInfos.Count > 0)
{
sw.WriteLine("Searched Time: " + priceInfos[0].SoldTime.ToShortDateString() + " => " + priceInfos[priceInfos.Count - 1].SoldTime.ToShortDateString());
Console.WriteLine("Searched Time: " + priceInfos[0].SoldTime.ToShortDateString() + " => " + priceInfos[priceInfos.Count - 1].SoldTime.ToShortDateString());
}
foreach (var priceInfo in priceInfos)
{
sw.WriteLine(priceInfo.ToJson());
Console.WriteLine(priceInfo.ToString());
}
}
}
}
public void AddResult(IScrapper scrapper, ScrapperRunnerConfiguration configuration, bool alertValue)
{
var resultKvp = new KeyValuePair <ScrapperRunnerConfiguration, bool>(configuration, alertValue);
var key = scrapper.GetType().Name;
if (_scrapperRunResults.ContainsKey(key))
{
_scrapperRunResults[key].Add(resultKvp);
}
else
{
_scrapperRunResults.Add(key, new List <KeyValuePair <ScrapperRunnerConfiguration, bool> > {
resultKvp
});
}
}
public OtodomFeedService(
ConfigHelper configHelper,
ILogger log,
IScrapper otoDomScrapper,
IOtoDomRepository otoDomRepository,
IMapper mapper)
{
this.log = log;
this.otoDomScrapper = otoDomScrapper;
this.otoDomRepository = otoDomRepository;
this.mapper = mapper;
if (configHelper == null || configHelper.OtodomFeedJobConfig == null)
{
throw new OtodomServiceException($"Missing configuration provider for {serviceName}.");
}
otodomUrl = configHelper.OtodomFeedJobConfig.AllOffersUrl;
if (string.IsNullOrEmpty(otodomUrl))
{
throw new OtodomServiceException($"Missing configuration section [OtodomFeedJob:otodomUrlAllOffers] for {serviceName}.");
}
}
public ScrapperServiceMock(IScrapper scrapper,
IEventRepository eventRepository,
IEventDateRepository eventDateRepository) : base(scrapper, eventRepository, eventDateRepository)
{
}
private void handleCompleted(WebBrowser browser, /*ProgressBar progress,*/
object sender, DownloadStringCompletedEventArgs e, IScrapper scrapper)
{
if (e.Error != null)
{
browser.NavigateToString("Error occured: " + e.Error.ToString());
}
else
{
if (e.Cancelled)
{
browser.NavigateToString("Request was cancelled");
}
else
{
var searchResults = scrapper.Scrape(e.Result);
browser.NavigateToString(GetHtml(searchResults));
}
}
}
public void AddConfigurations(IScrapper scrapper, IAlertConfiguration config)
{
AddConfigurations(scrapper, new ScrapperRunnerConfiguration(config));
}
public UpdatedVesselFactory(IScrapper scrapper, IUpdatingProgress progress)
{
_scrapper = scrapper;
_progress = progress;
}