Esempio n. 1
0
 public ScrapersManager(IInputDataProvider inputDataProvider, IDataHandler <Person> dataHandler, IScrapper <Person> scrapper, ILog log)
 {
     _inputDataProvider = inputDataProvider;
     _dataHandler       = dataHandler;
     _scrapper          = scrapper;
     _log = log;
 }
Esempio n. 2
0
        public ScrapperModule(IScrapperConfiguration scrapperConfiguration, IScrapper scrapper,
                              IEnumerable <IParser> parsers) : base("Radio")
        {
            _scrapperConfiguration = scrapperConfiguration;
            _parsers = parsers;

            Get["/Get/{radio}"] = parameters =>
            {
                if (!TryGetRadio((string)parameters.radio, out var radioConfiguration))
                {
                    throw new NotFoundErrorException($"Radio {(string) parameters.radio} not found");
                }

                var parser = GetParser(radioConfiguration.Name);

                var song = scrapper.Scrap(radioConfiguration.Uri, parser);

                return(Response.AsJson(song));
            };

            Get["/GetAsync/{radio}", true] = async(parameters, ct) =>
            {
                if (!TryGetRadio((string)parameters.radio, out var radioConfiguration))
                {
                    throw new NotFoundErrorException($"Radio {(string)parameters.radio} not found");
                }

                var parser = GetParser(radioConfiguration.Name);

                var song = await scrapper.ScrapAsync(radioConfiguration.Uri, parser);

                return(Response.AsJson(song));
            };
        }
        static void Main(string[] args)
        {
            IScrapper scrapper        = null;
            string    entryUrl        = null;
            string    outputDirectory = null;

            Parser.Default.ParseArguments <Options>(args)
            .WithParsed(o => {
                switch (o.Type)
                {
                case PodcastType.HowToSaveMoney:
                    scrapper = new HowToSaveMoney.Scrapper.Scrapper();
                    entryUrl = "https://jakoszczedzacpieniadze.pl/podcast";
                    break;

                case PodcastType.ForeignDailyNews:
                    scrapper = new ForeignDailyNews.Scrapper.Scrapper();
                    entryUrl = "http://www.dzialzagraniczny.pl/category/podcasty/";
                    break;

                default:
                    throw new System.ArgumentNullException(nameof(Options.Type));
                }

                outputDirectory = string.IsNullOrEmpty(o.OutputDirectory) ? DefaultOutput : o.OutputDirectory;
            });

            if (scrapper != null)
            {
                var downloader = new Downloader(scrapper, outputDirectory, entryUrl);
                downloader.Run().GetAwaiter().GetResult();
            }
        }
 public DownloadTask(IScrapper <State, University> universities,
                     IScrapper <University, Specialty> specialties,
                     IScrapper <Specialty, Abiturient> abiturients)
 {
     _universities = universities;
     _specialties  = specialties;
     _abiturients  = abiturients;
 }
 public Downloader(IScrapper scrapper, string outputPath, string entryUrl)
 {
     _scrapper              = scrapper;
     _outputPath            = outputPath;
     _entryUrl              = entryUrl;
     _articlesCacheFilePath = Path.Combine(_outputPath, "articles.txt");
     _downloadUrlsFilePath  = Path.Combine(_outputPath, "download.txt");
 }
Esempio n. 6
0
 public ScrapperService(
     IScrapper scrapper,
     IEventRepository eventRepository,
     IEventDateRepository eventDateRepository
     )
 {
     _eventRepository     = eventRepository;
     _eventDateRepository = eventDateRepository;
     _scrapper            = scrapper;
 }
Esempio n. 7
0
        static async Task MainAsync(IScrapper scrapper, ILogger logger)
        {
            try
            {
                await scrapper.RetreiveDataAsync();

                logger.LogInformation("COMPLETED.");
            }
            catch (Exception ex)
            {
                logger.LogCritical(ex, "Unexpected error.Exit.");
                Console.ReadKey();
            }
        }
        public ChapterSelector(IScrapper scrapper)
        {
            _scrapper = scrapper;
            InitializeComponent();

            // MyListView.ItemsSource = scrapper.SelectedManga.Chapters;
            ChapterSelectorViewModel =
                new ChapterSelectorViewModel(scrapper.SelectedManga, scrapper.SelectedManga.Chapters)
            {
                IsFavorite = _scrapper.SelectedManga.IsFavorite
            };

            BindingContext = ChapterSelectorViewModel;
        }
Esempio n. 9
0
 public void AddConfigurations(IScrapper scrapper, ScrapperRunnerConfiguration runnerConfig)
 {
     if (_scrapperConfigs.ContainsKey(scrapper))
     {
         _scrapperConfigs[scrapper].Add(runnerConfig);
     }
     else
     {
         _scrapperConfigs.Add(scrapper,
                              new HashSet <ScrapperRunnerConfiguration> {
             runnerConfig
         });
     }
 }
Esempio n. 10
0
        static void Main(string[] args)
        {
            if (args.Count() == 0)
            {
                Console.WriteLine("pricer location <hemnet|booli> <area>");
                Console.WriteLine("pricer get <location_id> <type> <timespan> <filepath> <hemnet|booli>");
                return;
            }

            if (args.Count() == 3)
            {
                var task = ScrapperFactory.GetLocations(args[1], args[2]);
                Console.WriteLine("Waiting result from providers...");
                task.Wait();
                var locations = task.Result;
                Console.WriteLine(String.Join("\n", locations.Select(x => x.GetId() + "\t" + x.GetName() + "\t" + x.GetAreaName()).ToArray()));
            }

            if (args.Count() == 6)
            {
                List <string> locationIds = args[1].Split(";").ToList();
                List <string> type        = args[2].Split(";").ToList();

                IScrapper scrapper = ScrapperFactory.GetScrapper(args[5], locationIds, type, args[3]);
                var       task     = scrapper.DoRequest();
                Console.WriteLine("Waiting result from providers...");
                task.Wait();

                var priceInfos = task.Result;

                using (StreamWriter sw = new StreamWriter(args[4]))
                {
                    sw.WriteLine("Total Object: " + priceInfos.Count);
                    Console.WriteLine("Total Object: " + priceInfos.Count);

                    if (priceInfos.Count > 0)
                    {
                        sw.WriteLine("Searched Time: " + priceInfos[0].SoldTime.ToShortDateString() + " => " + priceInfos[priceInfos.Count - 1].SoldTime.ToShortDateString());
                        Console.WriteLine("Searched Time: " + priceInfos[0].SoldTime.ToShortDateString() + " => " + priceInfos[priceInfos.Count - 1].SoldTime.ToShortDateString());
                    }

                    foreach (var priceInfo in priceInfos)
                    {
                        sw.WriteLine(priceInfo.ToJson());
                        Console.WriteLine(priceInfo.ToString());
                    }
                }
            }
        }
Esempio n. 11
0
            public void AddResult(IScrapper scrapper, ScrapperRunnerConfiguration configuration, bool alertValue)
            {
                var resultKvp = new KeyValuePair <ScrapperRunnerConfiguration, bool>(configuration, alertValue);
                var key       = scrapper.GetType().Name;

                if (_scrapperRunResults.ContainsKey(key))
                {
                    _scrapperRunResults[key].Add(resultKvp);
                }
                else
                {
                    _scrapperRunResults.Add(key, new List <KeyValuePair <ScrapperRunnerConfiguration, bool> > {
                        resultKvp
                    });
                }
            }
Esempio n. 12
0
        public OtodomFeedService(
            ConfigHelper configHelper,
            ILogger log,
            IScrapper otoDomScrapper,
            IOtoDomRepository otoDomRepository,
            IMapper mapper)
        {
            this.log              = log;
            this.otoDomScrapper   = otoDomScrapper;
            this.otoDomRepository = otoDomRepository;
            this.mapper           = mapper;

            if (configHelper == null || configHelper.OtodomFeedJobConfig == null)
            {
                throw new OtodomServiceException($"Missing configuration provider for {serviceName}.");
            }

            otodomUrl = configHelper.OtodomFeedJobConfig.AllOffersUrl;

            if (string.IsNullOrEmpty(otodomUrl))
            {
                throw new OtodomServiceException($"Missing configuration section [OtodomFeedJob:otodomUrlAllOffers] for {serviceName}.");
            }
        }
Esempio n. 13
0
 public ScrapperServiceMock(IScrapper scrapper,
                            IEventRepository eventRepository,
                            IEventDateRepository eventDateRepository) : base(scrapper, eventRepository, eventDateRepository)
 {
 }
Esempio n. 14
0
        private void handleCompleted(WebBrowser browser, /*ProgressBar progress,*/
            object sender, DownloadStringCompletedEventArgs e, IScrapper scrapper)
        {
            if (e.Error != null)
            {
                browser.NavigateToString("Error occured: " + e.Error.ToString());
            }
            else
            {
                if (e.Cancelled)
                {
                    browser.NavigateToString("Request was cancelled");
                }
                else
                {
                    var searchResults = scrapper.Scrape(e.Result);

                    browser.NavigateToString(GetHtml(searchResults));
                }
            }
        }
Esempio n. 15
0
 public void AddConfigurations(IScrapper scrapper, IAlertConfiguration config)
 {
     AddConfigurations(scrapper, new ScrapperRunnerConfiguration(config));
 }
Esempio n. 16
0
 public UpdatedVesselFactory(IScrapper scrapper, IUpdatingProgress progress)
 {
     _scrapper = scrapper;
     _progress = progress;
 }