public FactoryModule(
string factoryAddress,
FetcherOptions fetcherOptions)
{
_factoryAddress = factoryAddress;
_fetcherOptions = fetcherOptions;
}
public CustomerController(IConfiguration configuration, IOptions <FetcherOptions> opt, IUserApiClient api)
{
this._api = api;
this._config = configuration;
this._opt = opt.Value;
}
public Fetcher(FetcherOptions options, Func <FetchTarget, Task <FetchResult> > transform = null)
{
_options = options;
transform = transform ?? Fetch;
_input = new TransformBlock <FetchTarget, FetchResult>(transform, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = -1
});
_output = new BroadcastBlock <FetchResult>(x => x);
var linkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
_input.LinkTo(_output, linkOptions);
}
protected Crawler(IDocumentFactory documentFactory, IKeyValueStore <string, Result> store, IKeyValueStore <string, FetchTarget> frontier)
{
_store = store;
_frontier = frontier;
var fetcherOptions = new FetcherOptions
{
UserAgent = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36",
};
var parserOptions = new ParserOptions
{
};
var scraperOptions = new ScraperOptions
{
};
var extractorOptions = new ExtractorOptions
{
};
//var storerOptions = new StorerOptions
//{
//};
var builderOptions = new BuilderOptions
{
};
var providerOptions = new ProviderOptions
{
};
//var dispatcherOptions = new DispatcherOptions
//{
//};
Fetcher = new Fetcher(fetcherOptions);
Parser = new Parser(parserOptions, documentFactory);
Scraper = new Scraper(scraperOptions);
Extractor = new Extractor(extractorOptions);
Storer = new Storer(store);
Builder = new Builder(builderOptions);
Provider = new Provider(providerOptions, store, frontier);
Dispatcher = new Dispatcher();
Fetcher.SendTo(Parser, x => x.StatusCode == System.Net.HttpStatusCode.OK);
Parser.SendTo(Scraper);
Parser.SendTo(Extractor);
Fetcher.SendTo(Builder, x => x.StatusCode == System.Net.HttpStatusCode.OK);
Scraper.SendTo(Builder);
Extractor.SendTo(Builder);
Builder.SendTo(Storer);
//Storer.LinkTo(new ActionBlock<Result>(x =>
//{
//}));
Builder.SendTo(Provider);
Provider.SendTo(Dispatcher, x => x != null);
Dispatcher.SendTo(Fetcher);
}