public MainForm()
{
InitializeComponent();
pageParser = new PageParser(new PageDownloader());
codeGenerator = new CodeGenerator();
}
public static ISiteCrawler Create(SiteParameter siteParameter)
{
if (siteParameter == null)
{
throw new ArgumentNullException(nameof(siteParameter));
}
string dataServiceName = GetValueOrDefault(siteParameter.CustomProcessors, "IDataService");
string htmlReaderName = GetValueOrDefault(siteParameter.CustomProcessors, "IHtmlReader");
string pageParserName = GetValueOrDefault(siteParameter.CustomProcessors, "IPageParser");
string itemReaderName = GetValueOrDefault(siteParameter.CustomProcessors, "IItemReader");
string pageReaderName = GetValueOrDefault(siteParameter.CustomProcessors, "IPageReader");
IHtmlReader htmlReader = Container.Resolve <IHtmlReader>(htmlReaderName);
ParameterOverride htmlReaderParameter = new ParameterOverride("htmlReader", htmlReader);
ParameterOverride siteParameterParameter = new ParameterOverride("siteParameter", siteParameter);
IItemReader itemReader = Container.Resolve <IItemReader>(itemReaderName, siteParameterParameter);
ParameterOverride itemReaderParameter = new ParameterOverride("itemReader", itemReader);
IPageReader pageReader = Container.Resolve <IPageReader>(pageReaderName, siteParameterParameter, htmlReaderParameter, itemReaderParameter);
IPageParser pageParser = Container.Resolve <IPageParser>(pageParserName, siteParameterParameter, htmlReaderParameter);
return(new GeneralSiteCrawler(pageReader, pageParser));
}
public Crawler(IHttpClientWrapper client, IValidator validator, IPageParser parser)
{
_client = client;
_validator = validator;
_parser = parser;
// set default downloading properties for crawler
DownloadingGroupSize = 100;
}
public Spider(IUriMapper uriMapper, IUriTracker uriTracker, IPageLoader pageLoader, IPageArchive pageArchive, IPageParser pageParser, ILogger logger)
{
UriMapper = uriMapper;
UriTracker = uriTracker;
PageLoader = pageLoader;
PageArchive = pageArchive;
PageParser = pageParser;
Logger = logger;
}
public BadmintonLtPlayersProvider(
IPageParser <PlayerClubViewModel> playerClubsPageParser = null,
IPageParser <PlayerViewModel> playersPageParser = null)
{
_playersPageParser = playersPageParser
?? new BadmintonLtPlayersPageParser();
_playerClubsPageParser = playerClubsPageParser
?? new BadmintonLtClubsPageParser();
}
public GeneralSiteCrawler(SiteParameter siteParameter)
{
IItemReader itemReader = new RegexItemReader(siteParameter);
IHtmlReader htmlReader = new HttpClientReader();
this.pageReader = new SequentialPageReader(siteParameter, htmlReader, itemReader);
this.pageParser = new RegexPageParser(siteParameter, htmlReader);
}
public DownloadExpression(IAppLog appLog, IPageParser parser, IAppController appController, IFileDownloader downloader, IComicPath comicPath, IFileProxy file, IWebClientProxy webClient, IAppSettings appSettings, IComicStore comicStore)
: base(appLog, webClient)
{
this.parser = parser;
this.comicStore = comicStore;
this.appSettings = appSettings;
this.file = file;
this.comicPath = comicPath;
this.downloader = downloader;
this.appController = appController;
}
public GeneralSiteCrawler(IPageReader pageReader, IPageParser pageParser, IDataService dataService)
{
this.pageReader = pageReader ?? throw new ArgumentNullException(nameof(pageReader));
this.pageParser = pageParser ?? throw new ArgumentNullException(nameof(pageParser));
this.dataService = dataService ?? throw new ArgumentNullException(nameof(dataService));
this.pageParser.SetErrorHandler((url, exception) =>
this.dataService.AddLog(new CrawlerLog
{
Url = url,
LogTime = DateTime.Now,
Message = exception.Message
}));
}
/// <summary>
/// Initializes a new instance of the <see cref="StatementParser" /> class.
/// </summary>
/// <param name="tokenSplitter">The token splitter.</param>
/// <param name="whereParser">The where parser.</param>
/// <param name="pageParser">The page parser.</param>
/// <param name="orderByParser">The order by parser.</param>
/// <param name="selectParser">The select parser.</param>
public StatementParser(
[NotNull] ISplitter <TokenSplitterResponse> tokenSplitter,
[NotNull] IWhereParser whereParser,
[NotNull] IPageParser pageParser,
[NotNull] IOrderByParser orderByParser,
[NotNull] ISelectParser selectParser)
{
this.tokenSplitter = tokenSplitter;
this.whereParser = whereParser;
this.pageParser = pageParser;
this.orderByParser = orderByParser;
this.selectParser = selectParser;
}
public UrlProcessService(
IEntryRepository repository, IUnitOfWork unitOfWork,
AudioDownloader downloader,
IPageParser parser,
IYouTubeParser youTubeParser,
ILogger <UrlProcessService> logger, IRealTimeUpdater realtimeUpdater,
IMapper mapper) : base(logger, realtimeUpdater, mapper)
{
_repository = repository;
_unitOfWork = unitOfWork;
_downloader = downloader;
_parser = parser;
_youTubeParser = youTubeParser;
}
public DemotivatorApi(string domainUrl)
{
if (string.IsNullOrEmpty(domainUrl))
{
throw new ArgumentNullException(nameof(domainUrl), "Adres strony demotywatorów nie może być pusty");
}
var demotivatorParser = new DemotivatorParser(new DemotivatorBuilder(), domainUrl);
var slideDemotivatorParser = new DemotivatorSlideParser(new DemotivatorSlideBuilder());
var videoDemotivatorParser = new DemotivatorVideoParser(domainUrl);
this.PageParser = new PageParser(demotivatorParser, slideDemotivatorParser, videoDemotivatorParser, domainUrl);
}
public GeneralSiteCrawler(SiteParameter siteParameter)
{
this.dataService = new DbDataService(CrawlerDbHelper.GetContext());
IItemReader itemReader = new RegexItemReader(siteParameter);
IHtmlReader htmlReader = new HttpClientReader();
this.pageReader = new SequentialPageReader(siteParameter, htmlReader, itemReader);
this.pageParser = new RegexPageParser(siteParameter, htmlReader);
this.pageParser.SetErrorHandler((url, exception) =>
this.dataService.AddLog(new CrawlerLog
{
Url = url,
LogTime = DateTime.Now,
Message = exception.Message
}));
}
public BrowserExtensionController(
ILogger <BrowserExtensionController> logger,
IPodcastRepository podcastRepository,
IHttpContextAccessor contextAccessor,
UserManager <ApplicationUser> userManager,
IConfiguration options,
IOptions <ChatSettings> chatSettings,
IRepository <UserRequest> userRequestRepository,
IUnitOfWork unitOfWork,
IPageParser parser) : base(contextAccessor, userManager, logger)
{
_podcastRepository = podcastRepository;
_options = options;
_chatSettings = chatSettings.Value;
_userRequestRepository = userRequestRepository;
_unitOfWork = unitOfWork;
this._parser = parser;
}
public VideoList(String site, String videoUrl)
{
InitializeComponent();
if (database == null)
{
database = new Common.Database();
favoriteVVM = database.getVideo();
}
this.site = site;
this.videoUrl = videoUrl;
// Get Parser
switch (this.site)
{
case "zuidazy":
pageParser = new ZUIDAZY();
break;
case "jikzy":
pageParser = new JIKZY();
break;
}
isFavoritePage = (videoUrl.CompareTo("FAVORITE") == 0) ? true : false;
listView.ItemsSource = videos;
// Bind events
listView.ItemSelected += ListView_ItemSelected;
listView.ItemAppearing += listView_ItemAppearing;
loadMoreButton.Clicked += LoadMoreButton_Clicked;
// Load Videos
if (isFavoritePage)
{
loadMoreButton.IsVisible = false;
foreach (var video in favoriteVVM)
{
video.Image = video.Favorite ? tool.LIKEURL : tool.DISLIKEURL;
videos.Add(video);
}
}
else
{
LoadData();
}
}
public VideoDetail(String site, String videoUrl)
{
InitializeComponent();
this.site = site;
this.videoUrl = videoUrl;
// Get Parser
switch (this.site)
{
case "zuidazy":
pageParser = new ZUIDAZY();
break;
case "jikzy":
pageParser = new JIKZY();
break;
}
videos = new ObservableCollection <Common.VideoViewModel>();
lstView.ItemsSource = videos;
lstView.ItemSelected += ListView_ItemSelected;
LoadData();
}
public DebugController(IOptions <StorageSettings> settings, IOptions <AppSettings> appSettings,
HubLifetimeManager <DebugHub> hub,
IConfiguration config,
IPageParser pageParser,
IOptions <HelpersSettings> helpersSettings,
IOptions <AudioFileStorageSettings> audioFileStorageSettings,
IOptions <ImageFileStorageSettings> imageFileStorageSettings,
IOptions <JwtIssuerOptions> jwtIssuerOptions,
IPushSubscriptionStore subscriptionStore,
IEntryRepository entryRepository,
UserManager <ApplicationUser> userManager,
ILogger <DebugController> logger,
IMapper mapper,
IPushNotificationService notificationService,
IPodcastRepository podcastRepository,
IUnitOfWork unitOfWork,
IHttpContextAccessor contextAccessor,
IMailSender mailSender) : base(contextAccessor, userManager, logger)
{
_appSettings = appSettings.Value;
_storageSettings = settings.Value;
_helpersSettings = helpersSettings.Value;
_audioFileStorageSettings = audioFileStorageSettings.Value;
_imageFileStorageSettings = imageFileStorageSettings.Value;
_jwtIssuerOptions = jwtIssuerOptions.Value;
_hub = hub;
_config = config;
_pageParser = pageParser;
_mapper = mapper;
_subscriptionStore = subscriptionStore;
_entryRepository = entryRepository;
_notificationService = notificationService;
_podcastRepository = podcastRepository;
_unitOfWork = unitOfWork;
_mailSender = mailSender;
}
public MatchExpression(IAppLog appLog, IWebClientProxy webClient, IPageParser parser) : base(appLog, webClient)
{
this.parser = parser;
}
/// <summary>
/// Constructor for <see cref="EaterMain"/>.
/// </summary>
public EaterMain(HttpClient client, IPageParser parser)
{
this.client = client;
this.parser = parser;
}
public PageParserBase(IPageParser nextParser)
{
NextParser = nextParser;
}
public PeekExpression(IAppLog appLog, IWebClientProxy webClient, IPageParser parser, IAppController appController) : base(appLog, webClient)
{
this.parser = parser;
this.appController = appController;
}
public static void UnregisterParser(string extension, IPageParser parser)
{
_parsers.Remove(extension);
}
public static void RegisterParser(string extension, IPageParser parser)
{
_parsers[extension] = parser;
}
public Parsers()
{
Chain = new RozhlasPrehrat2018PageParser(
new RozhlasPrehrat2017PageParser());
}
public HackerNewsPostsService(IHackerNewsServiceAgent hackerNewsServiceAgent, IPageParser pageParser, IHackerNewsPostItemValidator hackerNewsPostItemValidator)
{
_hackerNewsServiceAgent = hackerNewsServiceAgent;
_pageParser = pageParser;
_hackerNewsPostItemValidator = hackerNewsPostItemValidator;
}
public OnlinerConnector(ILoadEngine engine, IResponseParser parser, IOnlinerPageParser pageParser)
{
this.engine = engine;
this.parser = parser;
this.pageParser = pageParser;
}
public RozhlasPrehrat2017PageParser(IPageParser next = null) : base(next)
{
}
public Worker(INavigator navigator, IPageParser pageParser)
{
_navigator = navigator;
_pageParser = pageParser;
}
public GeneralSiteCrawler(IPageReader pageReader, IPageParser pageParser)
{
this.pageReader = pageReader ?? throw new ArgumentNullException(nameof(pageReader));
this.pageParser = pageParser ?? throw new ArgumentNullException(nameof(pageParser));
}
public PageParserTests()
{
_postSectionParser = new PostSectionParser();
_pageParser = new PageParser(_postSectionParser);
}
public VisitExpression CreateVisitExpression(IPageParser parser)
{
return(new VisitExpression(parser, WebClient.Object, AppLog.Object, ApplicationController.Object));
}
