public void GetCellAndConvertToEntity() { var doc = LoadDocument <IHtmlDocument>("Html", "ariva.prices.DE0007664039.html"); var dataSource = new DataSource(); dataSource.Vendor = "Ariva"; dataSource.Name = "Prices"; dataSource.Quality = 1; var descriptor = new PathCellDescriptor(); descriptor.Figure = "Price"; descriptor.Path = @"/BODY[0]/DIV[0]/DIV[1]/DIV[6]/DIV[1]/DIV[0]/DIV[0]/TABLE[0]/TBODY[0]"; descriptor.Column = new StringContainsLocator { HeaderSeriesPosition = 0, Pattern = "Letzter" }; descriptor.Row = new StringContainsLocator { HeaderSeriesPosition = 0, Pattern = "Frankfurt" }; descriptor.ValueFormat = new FormatColumn("value", typeof(double), "00,00") { ExtractionPattern = new Regex(@"([0-9,\.]+)") }; descriptor.Currency = "EUR"; var parser = DocumentProcessingFactory.CreateParser(doc, descriptor); var table = parser.ExtractTable(); Assert.That(table.Rows.Count, Is.EqualTo(1)); var value = table.Rows[0][0]; Assert.That(value, Is.EqualTo(134.356d)); var converter = DocumentProcessingFactory.CreateConverter(descriptor, dataSource, new[] { new Currency { Symbol = "EUR" } }); var series = converter.Convert(table, new Stock { Isin = "DE0007664039" }); var price = (Price)series.Single(); Assert.That(price.Stock.Isin, Is.EqualTo("DE0007664039")); Assert.That(((DayPeriod)price.Period).Day.Date, Is.EqualTo(DateTime.Today)); Assert.That(price.Source, Does.Contain("ariva").IgnoreCase.And.Contains("price").IgnoreCase); Assert.That(price.Timestamp.Date, Is.EqualTo(DateTime.Today)); Assert.That(price.Value, Is.EqualTo(134.356d)); Assert.That(price.Currency.Symbol, Is.EqualTo("EUR")); }
private void TryFetch() { if (myDocumentBrowser == null) { return; } myData.Clear(); var descriptors = mySelectedSource.Figures .Cast <IPathDescriptor>() .Where(f => f.Figure == myFigureType.Name); foreach (var descriptor in descriptors) { try { ILocatorMacroResolver resolver = new StockMacroResolver(Stock); if (CustomResolverCreator != null) { resolver = CustomResolverCreator(resolver); } myDocumentBrowser.Navigate(DocumentType.Html, mySelectedSource.Location, resolver); var htmlDocument = ( IHtmlDocument )myDocumentBrowser.Document; // Mark the part of the document described by the FigureDescriptor to have a preview var cell = ( HtmlElementAdapter )MarkupFactory.FindElementByDescriptor(htmlDocument, descriptor); if (cell != null) { cell.Element.ScrollIntoView(false); } var marker = MarkupFactory.CreateMarker(descriptor); marker.Mark(cell); // already extract data here to check for format issues etc var parser = DocumentProcessingFactory.CreateParser(htmlDocument, descriptor); var table = parser.ExtractTable(); var converter = DocumentProcessingFactory.CreateConverter(descriptor, mySelectedSource, CurrenciesLut.Currencies); var series = converter.Convert(table, Stock); myData.AddRange(series); // we found s.th. with this format // -> skip alternative formats break; } catch (Exception ex) { ex.Data["Figure"] = myFigureType.Name; ex.Data["DataSource.Vendor"] = mySelectedSource.Vendor; ex.Data["DataSource.Name"] = mySelectedSource.Name; ex.Data["Location"] = mySelectedSource.Location.ToString(); ex.Data["FigureDescriptor"] = descriptor.GetType().FullName; if (ThrowOnError) { throw new Exception("Failed to extract data from datasource", ex); } else { myLogger.Error(ex, "Failed to fetch '{0}' from site {1}", myFigureType.Name, mySelectedSource.Name); } } } }
public void GetSeriesAndConvertToEntities() { var doc = LoadDocument <IHtmlDocument>("Html", "ariva.fundamentals.DE0005190003.html"); var dataSource = new DataSource(); dataSource.Vendor = "Ariva"; dataSource.Name = "Fundamentals"; dataSource.Quality = 1; var descriptor = new PathSeriesDescriptor(); descriptor.Figure = "Dividend"; descriptor.Path = @"/BODY[0]/DIV[5]/DIV[0]/DIV[1]/TABLE[7]/TBODY[0]"; descriptor.Orientation = SeriesOrientation.Row; descriptor.ValuesLocator = new StringContainsLocator { HeaderSeriesPosition = 0, Pattern = "Dividendenausschüttung" }; descriptor.ValueFormat = new FormatColumn("value", typeof(double), "00,00") { InMillions = true }; descriptor.TimesLocator = new AbsolutePositionLocator { HeaderSeriesPosition = 0, SeriesPosition = 1 }; descriptor.TimeFormat = new FormatColumn("year", typeof(int), "00000000"); descriptor.Excludes.Add(0); var parser = DocumentProcessingFactory.CreateParser(doc, descriptor); var table = parser.ExtractTable(); Assert.AreEqual(6, table.Rows.Count); Assert.AreEqual(350000000d, table.Rows[0][0]); Assert.AreEqual(351000000d, table.Rows[1][0]); Assert.AreEqual(392000000d, table.Rows[2][0]); Assert.AreEqual(419000000d, table.Rows[3][0]); Assert.AreEqual(424000000d, table.Rows[4][0]); Assert.AreEqual(458000000d, table.Rows[5][0]); Assert.AreEqual(2001, table.Rows[0][1]); Assert.AreEqual(2002, table.Rows[1][1]); Assert.AreEqual(2003, table.Rows[2][1]); Assert.AreEqual(2004, table.Rows[3][1]); Assert.AreEqual(2005, table.Rows[4][1]); Assert.AreEqual(2006, table.Rows[5][1]); var stock = new Stock { Isin = "DE0007664039" }; stock.Company = new Company { Name = "Volkswagen" }; stock.Company.Stocks.Add(stock); var converter = DocumentProcessingFactory.CreateConverter(descriptor, dataSource, Enumerable.Empty <Currency>()); var series = converter.Convert(table, stock).Cast <Dividend>().ToList(); foreach (var dividend in series) { Assert.That(dividend.Company.Stocks.First().Isin, Is.EqualTo("DE0007664039")); Assert.That(dividend.Period, Is.InstanceOf <YearPeriod>()); Assert.That(dividend.Source, Does.Contain("ariva").IgnoreCase.And.Contains("fundamentals").IgnoreCase); Assert.That(dividend.Timestamp.Date, Is.EqualTo(DateTime.Today)); // Descriptor does not provide static currency Assert.That(dividend.Currency, Is.Null); } Assert.That(series[0].Period, Is.EqualTo(new YearPeriod(2001))); Assert.That(series[0].Value, Is.EqualTo(350000000d)); Assert.That(series[1].Period, Is.EqualTo(new YearPeriod(2002))); Assert.That(series[1].Value, Is.EqualTo(351000000d)); Assert.That(series[2].Period, Is.EqualTo(new YearPeriod(2003))); Assert.That(series[2].Value, Is.EqualTo(392000000d)); Assert.That(series[3].Period, Is.EqualTo(new YearPeriod(2004))); Assert.That(series[3].Value, Is.EqualTo(419000000d)); Assert.That(series[4].Period, Is.EqualTo(new YearPeriod(2005))); Assert.That(series[4].Value, Is.EqualTo(424000000d)); Assert.That(series[5].Period, Is.EqualTo(new YearPeriod(2006))); Assert.That(series[5].Value, Is.EqualTo(458000000d)); }