public void MtgoDataIsParsedCorrectly() { _magicOnlineParser = new MagicOnlineTournamentParser(); //This is a live test, the handler relies on this data being accurate. If the site changes format, the parser will no longer work. //All Tests combined so they'll run quicker, since we need to go out and get data from WOTC to run the tests. var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15"; _resultSet = _magicOnlineParser.LoadTournamentResults(url); //Pilots Assert.AreEqual("sans", _resultSet.Decks[0].Pilot, "First Pilot is wrong."); Assert.AreEqual("giorno211", _resultSet.Decks[10].Pilot, "Last Pilot is wrong."); Assert.AreEqual(11, _resultSet.Decks.Select(x => x.Pilot).Distinct().ToList().Count, "11 individual pilots should be returned."); //Results Assert.AreEqual("4-0", _resultSet.Decks[0].Record, "First tourney record is wrong."); Assert.AreEqual("3-1", _resultSet.Decks[10].Record, "Last tourney record is wrong."); //Decks Assert.AreEqual(11, _resultSet.Decks.Count, "Incorrect # of decks returned."); //Tournament Information Assert.AreEqual("STANDARD DAILY #8582313 ON 08/14/2015", _resultSet.TournamentInformation.ToUpper(), "Incorrect Tourney Information."); //MTGO Flag Assert.IsTrue(_resultSet.IsMtgoResult, "MTGO Flag not getting set correctly."); }
/// <exception cref="ArgumentNullException"> /// <paramref name="htmlLoader" /> or /// <paramref name="parser" /> or /// <paramref name="backupProcessor" /> or /// <paramref name="httpDownloadManager" /> or /// <paramref name="dataService" /> or /// <paramref name="fileSystemProxy" /> is <see langword="null" />.</exception> internal PluralsightCatalog(IHtmlLoader htmlLoader, ITrainingCatalogParser <PluralsightCategory, PluralsightCourse, PluralsightAuthor> parser, ICatalogBackupProcessor <PluralsightCategory, PluralsightCourse, PluralsightAuthor> backupProcessor, IHttpDownloadManager httpDownloadManager, IPluralsightDataService dataService, IFileSystemProxy fileSystemProxy) : base(httpDownloadManager, fileSystemProxy) { if (htmlLoader == null) { throw new ArgumentNullException("htmlLoader"); } if (parser == null) { throw new ArgumentNullException("parser"); } if (backupProcessor == null) { throw new ArgumentNullException("backupProcessor"); } if (dataService == null) { throw new ArgumentNullException("dataService"); } _htmlLoader = htmlLoader; _parser = parser; _backupProcessor = backupProcessor; _dataService = dataService; }
protected HtmlDocumentParser( IHtmlLoader htmlLoader, IRecipeNameGenerator nameGenerator, INameUnicityOverseer nameUnicityOverseer, Func <CultureInfo, IActionDetector> actionDetectorFactory, Func <CultureInfo, ITimerDetector> timerDetectorFactory, Func <CultureInfo, IMeasureUnitDetector> measureUnitDetectorFactory, Func <CultureInfo, ILanguageHelper> languageHelperFactory, Func <CultureInfo, ISubrecipeRepository> subrecipeRepositoryFactory) { this._htmlLoader = htmlLoader; this._nameGenerator = nameGenerator; this._nameUnicityOverseer = nameUnicityOverseer; this._actionDetectorFactory = actionDetectorFactory; this._timerDetectorFactory = timerDetectorFactory; this._measureUnitDetectorFactory = measureUnitDetectorFactory; this._languageHelperFactory = languageHelperFactory; this._subrecipeRepositoryFactory = subrecipeRepositoryFactory; if (this._htmlLoader != null) { this._htmlLoader.Initialize(); } this.Base = new TBase(); // TODO Localize and put somewhere else this._quantityExpression = new Regex(@"[\xbc-\xbe\w]+[\xbc-\xbe\w'’,./]*", RegexOptions.Compiled); this._quantityRangeExpression = new Regex(@"((ou|à) [\xbc-\xbe0-9]+[\xbc-\xbe0-9'’,./]* )", RegexOptions.Compiled); this._ingredientExpression = new Regex(@"(?<=[a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%] de | d'| d’)([a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%]+)(, [,\w\s]+)*", RegexOptions.Compiled); this._ingredientFullExpression = new Regex(@"([a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%]+)(, [,\w\s]+)*", RegexOptions.Compiled); this._ingredientUnitExpression = new Regex(@"(?<=[a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%])([a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%]+)(, [,\w\s]+)*", RegexOptions.Compiled); }
public void MtgoDataIsParsedCorrectly() { _magicOnlineParser = new MagicOnlineTournamentParser(); //This is a live test, the handler relies on this data being accurate. If the site changes format, the parser will no longer work. //All Tests combined so they'll run quicker, since we need to go out and get data from WOTC to run the tests. var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15"; _resultSet = _magicOnlineParser.LoadTournamentResults(url); //Pilots Assert.AreEqual("sans", _resultSet.Decks[0].Pilot, "First Pilot is wrong."); Assert.AreEqual("giorno211", _resultSet.Decks[10].Pilot, "Last Pilot is wrong."); Assert.AreEqual(11, _resultSet.Decks.Select(x => x.Pilot).Distinct().ToList().Count, "11 individual pilots should be returned."); //Results Assert.AreEqual("4-0", _resultSet.Decks[0].Record, "First tourney record is wrong."); Assert.AreEqual("3-1", _resultSet.Decks[10].Record, "Last tourney record is wrong."); //Decks Assert.AreEqual(11, _resultSet.Decks.Count, "Incorrect # of decks returned."); //Tournament Information Assert.AreEqual("STANDARD DAILY #8582313 ON 08/14/2015", _resultSet.TournamentInformation.ToUpper(), "Incorrect Tourney Information."); //MTGO Flag Assert.IsTrue(_resultSet.IsMtgoResult, "MTGO Flag not getting set correctly."); }
public List<TournamentResults> Standard(string startDate, int numberOfDays) { _parserBl = new MagicOnlineTournamentParser(); var tournaments = new List<TournamentResults>(); var urlBaseString = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-"; var actualStartDate = DateTime.Parse(startDate); for (var x = 0; x < numberOfDays; x++) { try { var newDate = actualStartDate.AddDays(x); var url = urlBaseString + newDate.Year + "-" + newDate.Month.ToString("d2") + "-" + newDate.Day.ToString("00"); tournaments.Add(_parserBl.LoadTournamentResults(url)); } catch { //Tournament page doesn't exist } } return tournaments; }
public EtlController(IHtmlExtractor htmlExtractor, IHtmlTransformer htmlTransformer, IHtmlLoader htmlLoader, ReviewsDbContext reviewsDbContext) { _htmlExtractor = htmlExtractor; _htmlTransformer = htmlTransformer; _htmlLoader = htmlLoader; _reviewsDbContext = reviewsDbContext; }
public ActionResult Index() { _parserBl = new MagicOnlineTournamentParser(); var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15"; var viewModel = _parserBl.LoadTournamentResults(url); return View(viewModel); }
public ActionResult Index() { _parserBl = new MagicOnlineTournamentParser(); var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15"; var viewModel = _parserBl.LoadTournamentResults(url); return(View(viewModel)); }
public RicardoParser( IHtmlLoader htmlLoader, IRecipeNameGenerator nameGenerator, INameUnicityOverseer nameUnicityOverseer, Func <CultureInfo, IActionDetector> actionDetectorFactory, Func <CultureInfo, ITimerDetector> timerDetectorFactory, Func <CultureInfo, IMeasureUnitDetector> measureUnitDetectorFactory, Func <CultureInfo, ILanguageHelper> languageHelperFactory, Func <CultureInfo, ISubrecipeRepository> subrecipeRepositoryFactory) : base(htmlLoader, nameGenerator, nameUnicityOverseer, actionDetectorFactory, timerDetectorFactory, measureUnitDetectorFactory, languageHelperFactory, subrecipeRepositoryFactory) { }
public ArticleProvider(IHtmlCleaner cleaner, IHtmlLoader loader) { if (cleaner == null) { throw new ArgumentNullException("cleaner"); } if (loader == null) { throw new ArgumentNullException("loader"); } _cleaner = cleaner; _loader = loader; }
private async Task <ProductData> ExtractProductData(Product product) { IHtmlLoader htmlLoader = _htmlLoaderFactory.Get(product.Site); var cancelationSource = new CancellationTokenSource(); var document = await htmlLoader.LoadHtml(product.Url, product.Site, cancelationSource.Token); var priceParser = _priceParserFactory.Get <IDocument>(product.Site); var parserSettings = _configuration.GetSection(product.Site.Name).Get <ExtractorSettings>(); if (parserSettings == null) { throw new ArgumentException($"Не удалось найти настройки {nameof(ExtractorSettings)} в конфигурации для сайта {product.Site.Name}"); } return(await priceParser.Extract(document, parserSettings)); }
public List <TournamentResults> Standard(string startDate, int numberOfDays) { _parserBl = new MagicOnlineTournamentParser(); var tournaments = new List <TournamentResults>(); var urlBaseString = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-"; var actualStartDate = DateTime.Parse(startDate); for (var x = 0; x < numberOfDays; x++) { try { var newDate = actualStartDate.AddDays(x); var url = urlBaseString + newDate.Year + "-" + newDate.Month.ToString("d2") + "-" + newDate.Day.ToString("00"); tournaments.Add(_parserBl.LoadTournamentResults(url)); } catch { //Tournament page doesn't exist } } return(tournaments); }
public Spidy(IHtmlLoader htmlLoader, ISpidyParser parser) { _htmlLoader = htmlLoader; _parser = parser; }
public SiteCollectionProcessor(IHtmlLoader htmlLoader) { this.htmlLoader = htmlLoader; }
public PluralsightCatalog() : base(new HttpDownloadManager(), Shared.FileSystemProxy.Instance) { _htmlLoader = new HtmlLoader <HtmlAgilityPackHtmlContainer>(); _backupProcessor = new PluralsightBackupProcessor(); }
public RicardoCrawler(IHtmlLoader htmlLoader) : base(htmlLoader) { }
public FluentHtmlSelector(IHtmlLoader htmlLoader) { _document = new HtmlDocument(); _document.LoadHtml(htmlLoader.Load()); }
public void SetUp() { _htmlLoader = MockRepository.GenerateStub<IHtmlLoader>(); _htmlLoader.Stub(x => x.Load()).Return(HTML); }
public CrawlerContextConfigureOptions(IHtmlLoader loader, ResultWrappedHttpClient client) { _loader = loader; _client = client; }
protected SiteCrawler(IHtmlLoader htmlLoader) { this._htmlLoader = htmlLoader; this._htmlLoader.Initialize(); this.Base = new TBase(); }
public Crawler(IHttpClient http, IHtmlLoader html, Chain chain) { this._chain = chain; this._http = http; this._html = html; }
public Rzwde(IHtmlLoader uriLoader) { this.uriLoader = uriLoader; }