public FactoryModule(
     string factoryAddress,
     FetcherOptions fetcherOptions)
 {
     _factoryAddress = factoryAddress;
     _fetcherOptions = fetcherOptions;
 }
        public CustomerController(IConfiguration configuration, IOptions <FetcherOptions> opt, IUserApiClient api)
        {
            this._api    = api;
            this._config = configuration;

            this._opt = opt.Value;
        }
Exemple #3
0
        public Fetcher(FetcherOptions options, Func <FetchTarget, Task <FetchResult> > transform = null)
        {
            _options  = options;
            transform = transform ?? Fetch;

            _input = new TransformBlock <FetchTarget, FetchResult>(transform, new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism = -1
            });

            _output = new BroadcastBlock <FetchResult>(x => x);

            var linkOptions = new DataflowLinkOptions
            {
                PropagateCompletion = true
            };

            _input.LinkTo(_output, linkOptions);
        }
Exemple #4
0
        protected Crawler(IDocumentFactory documentFactory, IKeyValueStore <string, Result> store, IKeyValueStore <string, FetchTarget> frontier)
        {
            _store    = store;
            _frontier = frontier;

            var fetcherOptions = new FetcherOptions
            {
                UserAgent = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36",
            };

            var parserOptions = new ParserOptions
            {
            };

            var scraperOptions = new ScraperOptions
            {
            };

            var extractorOptions = new ExtractorOptions
            {
            };

            //var storerOptions = new StorerOptions
            //{
            //};

            var builderOptions = new BuilderOptions
            {
            };

            var providerOptions = new ProviderOptions
            {
            };

            //var dispatcherOptions = new DispatcherOptions
            //{
            //};


            Fetcher    = new Fetcher(fetcherOptions);
            Parser     = new Parser(parserOptions, documentFactory);
            Scraper    = new Scraper(scraperOptions);
            Extractor  = new Extractor(extractorOptions);
            Storer     = new Storer(store);
            Builder    = new Builder(builderOptions);
            Provider   = new Provider(providerOptions, store, frontier);
            Dispatcher = new Dispatcher();

            Fetcher.SendTo(Parser, x => x.StatusCode == System.Net.HttpStatusCode.OK);

            Parser.SendTo(Scraper);
            Parser.SendTo(Extractor);

            Fetcher.SendTo(Builder, x => x.StatusCode == System.Net.HttpStatusCode.OK);
            Scraper.SendTo(Builder);
            Extractor.SendTo(Builder);

            Builder.SendTo(Storer);

            //Storer.LinkTo(new ActionBlock<Result>(x =>
            //{
            //}));

            Builder.SendTo(Provider);
            Provider.SendTo(Dispatcher, x => x != null);
            Dispatcher.SendTo(Fetcher);
        }