public void MtgoDataIsParsedCorrectly()
{
_magicOnlineParser = new MagicOnlineTournamentParser();
//This is a live test, the handler relies on this data being accurate. If the site changes format, the parser will no longer work.
//All Tests combined so they'll run quicker, since we need to go out and get data from WOTC to run the tests.
var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15";
_resultSet = _magicOnlineParser.LoadTournamentResults(url);
//Pilots
Assert.AreEqual("sans", _resultSet.Decks[0].Pilot, "First Pilot is wrong.");
Assert.AreEqual("giorno211", _resultSet.Decks[10].Pilot, "Last Pilot is wrong.");
Assert.AreEqual(11, _resultSet.Decks.Select(x => x.Pilot).Distinct().ToList().Count, "11 individual pilots should be returned.");
//Results
Assert.AreEqual("4-0", _resultSet.Decks[0].Record, "First tourney record is wrong.");
Assert.AreEqual("3-1", _resultSet.Decks[10].Record, "Last tourney record is wrong.");
//Decks
Assert.AreEqual(11, _resultSet.Decks.Count, "Incorrect # of decks returned.");
//Tournament Information
Assert.AreEqual("STANDARD DAILY #8582313 ON 08/14/2015", _resultSet.TournamentInformation.ToUpper(), "Incorrect Tourney Information.");
//MTGO Flag
Assert.IsTrue(_resultSet.IsMtgoResult, "MTGO Flag not getting set correctly.");
}
/// <exception cref="ArgumentNullException">
/// <paramref name="htmlLoader" /> or
/// <paramref name="parser" /> or
/// <paramref name="backupProcessor" /> or
/// <paramref name="httpDownloadManager" /> or
/// <paramref name="dataService" /> or
/// <paramref name="fileSystemProxy" /> is <see langword="null" />.</exception>
internal PluralsightCatalog(IHtmlLoader htmlLoader,
ITrainingCatalogParser <PluralsightCategory, PluralsightCourse, PluralsightAuthor> parser,
ICatalogBackupProcessor <PluralsightCategory, PluralsightCourse, PluralsightAuthor> backupProcessor,
IHttpDownloadManager httpDownloadManager,
IPluralsightDataService dataService,
IFileSystemProxy fileSystemProxy)
: base(httpDownloadManager, fileSystemProxy)
{
if (htmlLoader == null)
{
throw new ArgumentNullException("htmlLoader");
}
if (parser == null)
{
throw new ArgumentNullException("parser");
}
if (backupProcessor == null)
{
throw new ArgumentNullException("backupProcessor");
}
if (dataService == null)
{
throw new ArgumentNullException("dataService");
}
_htmlLoader = htmlLoader;
_parser = parser;
_backupProcessor = backupProcessor;
_dataService = dataService;
}
protected HtmlDocumentParser(
IHtmlLoader htmlLoader,
IRecipeNameGenerator nameGenerator,
INameUnicityOverseer nameUnicityOverseer,
Func <CultureInfo, IActionDetector> actionDetectorFactory,
Func <CultureInfo, ITimerDetector> timerDetectorFactory,
Func <CultureInfo, IMeasureUnitDetector> measureUnitDetectorFactory,
Func <CultureInfo, ILanguageHelper> languageHelperFactory,
Func <CultureInfo, ISubrecipeRepository> subrecipeRepositoryFactory)
{
this._htmlLoader = htmlLoader;
this._nameGenerator = nameGenerator;
this._nameUnicityOverseer = nameUnicityOverseer;
this._actionDetectorFactory = actionDetectorFactory;
this._timerDetectorFactory = timerDetectorFactory;
this._measureUnitDetectorFactory = measureUnitDetectorFactory;
this._languageHelperFactory = languageHelperFactory;
this._subrecipeRepositoryFactory = subrecipeRepositoryFactory;
if (this._htmlLoader != null)
{
this._htmlLoader.Initialize();
}
this.Base = new TBase();
// TODO Localize and put somewhere else
this._quantityExpression = new Regex(@"[\xbc-\xbe\w]+[\xbc-\xbe\w'’,./]*", RegexOptions.Compiled);
this._quantityRangeExpression = new Regex(@"((ou|à) [\xbc-\xbe0-9]+[\xbc-\xbe0-9'’,./]* )", RegexOptions.Compiled);
this._ingredientExpression = new Regex(@"(?<=[a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%] de | d'| d’)([a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%]+)(, [,\w\s]+)*", RegexOptions.Compiled);
this._ingredientFullExpression = new Regex(@"([a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%]+)(, [,\w\s]+)*", RegexOptions.Compiled);
this._ingredientUnitExpression = new Regex(@"(?<=[a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%])([a-zA-Z0-9\u00C0-\u017F\s()'’\-\/%]+)(, [,\w\s]+)*", RegexOptions.Compiled);
}
public void MtgoDataIsParsedCorrectly()
{
_magicOnlineParser = new MagicOnlineTournamentParser();
//This is a live test, the handler relies on this data being accurate. If the site changes format, the parser will no longer work.
//All Tests combined so they'll run quicker, since we need to go out and get data from WOTC to run the tests.
var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15";
_resultSet = _magicOnlineParser.LoadTournamentResults(url);
//Pilots
Assert.AreEqual("sans", _resultSet.Decks[0].Pilot, "First Pilot is wrong.");
Assert.AreEqual("giorno211", _resultSet.Decks[10].Pilot, "Last Pilot is wrong.");
Assert.AreEqual(11, _resultSet.Decks.Select(x => x.Pilot).Distinct().ToList().Count, "11 individual pilots should be returned.");
//Results
Assert.AreEqual("4-0", _resultSet.Decks[0].Record, "First tourney record is wrong.");
Assert.AreEqual("3-1", _resultSet.Decks[10].Record, "Last tourney record is wrong.");
//Decks
Assert.AreEqual(11, _resultSet.Decks.Count, "Incorrect # of decks returned.");
//Tournament Information
Assert.AreEqual("STANDARD DAILY #8582313 ON 08/14/2015", _resultSet.TournamentInformation.ToUpper(), "Incorrect Tourney Information.");
//MTGO Flag
Assert.IsTrue(_resultSet.IsMtgoResult, "MTGO Flag not getting set correctly.");
}
public List<TournamentResults> Standard(string startDate, int numberOfDays)
{
_parserBl = new MagicOnlineTournamentParser();
var tournaments = new List<TournamentResults>();
var urlBaseString = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-";
var actualStartDate = DateTime.Parse(startDate);
for (var x = 0; x < numberOfDays; x++)
{
try
{
var newDate = actualStartDate.AddDays(x);
var url = urlBaseString + newDate.Year + "-" + newDate.Month.ToString("d2") + "-" + newDate.Day.ToString("00");
tournaments.Add(_parserBl.LoadTournamentResults(url));
}
catch
{
//Tournament page doesn't exist
}
}
return tournaments;
}
public EtlController(IHtmlExtractor htmlExtractor, IHtmlTransformer htmlTransformer, IHtmlLoader htmlLoader, ReviewsDbContext reviewsDbContext)
{
_htmlExtractor = htmlExtractor;
_htmlTransformer = htmlTransformer;
_htmlLoader = htmlLoader;
_reviewsDbContext = reviewsDbContext;
}
public ActionResult Index()
{
_parserBl = new MagicOnlineTournamentParser();
var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15";
var viewModel = _parserBl.LoadTournamentResults(url);
return View(viewModel);
}
public ActionResult Index()
{
_parserBl = new MagicOnlineTournamentParser();
var url = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-2015-08-15";
var viewModel = _parserBl.LoadTournamentResults(url);
return(View(viewModel));
}
public RicardoParser(
IHtmlLoader htmlLoader,
IRecipeNameGenerator nameGenerator,
INameUnicityOverseer nameUnicityOverseer,
Func <CultureInfo, IActionDetector> actionDetectorFactory,
Func <CultureInfo, ITimerDetector> timerDetectorFactory,
Func <CultureInfo, IMeasureUnitDetector> measureUnitDetectorFactory,
Func <CultureInfo, ILanguageHelper> languageHelperFactory,
Func <CultureInfo, ISubrecipeRepository> subrecipeRepositoryFactory)
: base(htmlLoader, nameGenerator, nameUnicityOverseer, actionDetectorFactory, timerDetectorFactory, measureUnitDetectorFactory, languageHelperFactory, subrecipeRepositoryFactory)
{
}
public ArticleProvider(IHtmlCleaner cleaner, IHtmlLoader loader)
{
if (cleaner == null)
{
throw new ArgumentNullException("cleaner");
}
if (loader == null)
{
throw new ArgumentNullException("loader");
}
_cleaner = cleaner;
_loader = loader;
}
private async Task <ProductData> ExtractProductData(Product product)
{
IHtmlLoader htmlLoader = _htmlLoaderFactory.Get(product.Site);
var cancelationSource = new CancellationTokenSource();
var document = await htmlLoader.LoadHtml(product.Url, product.Site, cancelationSource.Token);
var priceParser = _priceParserFactory.Get <IDocument>(product.Site);
var parserSettings = _configuration.GetSection(product.Site.Name).Get <ExtractorSettings>();
if (parserSettings == null)
{
throw new ArgumentException($"Не удалось найти настройки {nameof(ExtractorSettings)} в конфигурации для сайта {product.Site.Name}");
}
return(await priceParser.Extract(document, parserSettings));
}
public List <TournamentResults> Standard(string startDate, int numberOfDays)
{
_parserBl = new MagicOnlineTournamentParser();
var tournaments = new List <TournamentResults>();
var urlBaseString = "http://magic.wizards.com/en/articles/archive/mtgo-standings/standard-daily-";
var actualStartDate = DateTime.Parse(startDate);
for (var x = 0; x < numberOfDays; x++)
{
try
{
var newDate = actualStartDate.AddDays(x);
var url = urlBaseString + newDate.Year + "-" + newDate.Month.ToString("d2") + "-" + newDate.Day.ToString("00");
tournaments.Add(_parserBl.LoadTournamentResults(url));
}
catch
{
//Tournament page doesn't exist
}
}
return(tournaments);
}
public Spidy(IHtmlLoader htmlLoader, ISpidyParser parser)
{
_htmlLoader = htmlLoader;
_parser = parser;
}
public SiteCollectionProcessor(IHtmlLoader htmlLoader)
{
this.htmlLoader = htmlLoader;
}
public PluralsightCatalog()
: base(new HttpDownloadManager(), Shared.FileSystemProxy.Instance)
{
_htmlLoader = new HtmlLoader <HtmlAgilityPackHtmlContainer>();
_backupProcessor = new PluralsightBackupProcessor();
}
public RicardoCrawler(IHtmlLoader htmlLoader)
: base(htmlLoader)
{
}
public FluentHtmlSelector(IHtmlLoader htmlLoader)
{
_document = new HtmlDocument();
_document.LoadHtml(htmlLoader.Load());
}
public void SetUp()
{
_htmlLoader = MockRepository.GenerateStub<IHtmlLoader>();
_htmlLoader.Stub(x => x.Load()).Return(HTML);
}
public CrawlerContextConfigureOptions(IHtmlLoader loader, ResultWrappedHttpClient client)
{
_loader = loader;
_client = client;
}
protected SiteCrawler(IHtmlLoader htmlLoader)
{
this._htmlLoader = htmlLoader;
this._htmlLoader.Initialize();
this.Base = new TBase();
}
public Crawler(IHttpClient http, IHtmlLoader html, Chain chain)
{
this._chain = chain;
this._http = http;
this._html = html;
}
public Rzwde(IHtmlLoader uriLoader)
{
this.uriLoader = uriLoader;
}
