public GenericDatumProvider(IDocumentBrowser webScrapSC, DatumLocator locator, IFetchPolicy fetchPolicy, IResultPolicy resultPolicy)
{
myWebScrapSC = webScrapSC;
Locator = locator;
FetchPolicy = fetchPolicy;
ResultPolicy = resultPolicy;
}
private ParameterizedDatumLocator CreateLocator(DatumLocator locator, int siteIndex, string isin)
{
var validator = new ParameterizedDatumLocator(locator.Datum, locator.Sites[siteIndex]);
validator.Parameters["stock.isin"] = isin;
return(validator);
}
public static SingleResultValue <T> FetchSingle <T>(this DatumLocator datum, string isin)
{
var lut = new Dictionary <string, string>()
{
{ "stock.isin", isin }
};
var provider = CreateProvider(datum, new EvaluatorPolicy(lut));
return(provider.FetchSingle <T>());
}
/// <summary>
/// If locator with same name already exists it will be overwritten.
/// </summary>
public void Add(DatumLocator locator)
{
var existingLocator = myLocators.FirstOrDefault(l => l.Datum == locator.Datum);
if (existingLocator != null)
{
myLocators.Remove(existingLocator);
}
myLocators.Add(locator);
}
public static DataTable Fetch(this DatumLocator datum, string isin)
{
var lut = new Dictionary <string, string>()
{
{ "stock.isin", isin }
};
var provider = CreateProvider(datum, new EvaluatorPolicy(lut));
return(provider.Fetch().ResultTable);
}
public void ArivaSomePrices()
{
DatumLocator locator = new DatumLocator("StockPrices",
new Site("Ariva",
new Navigation(DocumentType.Html,
OS.CombinePaths(TestDataRoot, "Recognition", "ariva.prices.${stock.isin}.html")),
new PathTableFormat("Ariava.Prices",
"/BODY[0]/DIV[5]/DIV[0]/DIV[3]/DIV[0]/TABLE[${TableIndex}]",
new FormatColumn("date", typeof(DateTime), "dd.MM.yy"),
new FormatColumn("open", typeof(double), "000000,0000"),
new FormatColumn("close", typeof(double), "000000,0000"),
new FormatColumn("volume", typeof(int), "000000,0000"))
{
SkipRows = new int[] { 0, 1 },
SkipColumns = new int[] { 0, 3, 4 }
},
new DataContent("Euro")));
LookupPolicy fetchPolicy = new LookupPolicy();
fetchPolicy.Lut["${stock.isin}"] = "DE0005003404";
fetchPolicy.Lut["${TableIndex}"] = "0";
var webScrapSC = new ServiceProvider().Browser();
var provider = new GenericDatumProvider(webScrapSC, locator, fetchPolicy, null);
var result = provider.Fetch();
Assert.AreEqual("/BODY[0]/DIV[5]/DIV[0]/DIV[3]/DIV[0]/TABLE[${TableIndex}]", ((PathTableFormat)locator.Sites[0].Format).Path);
Assert.IsNotNull(result);
Assert.IsNotNull(result.ResultTable);
var table = result.ResultTable;
Assert.AreEqual(21, table.Rows.Count);
Assert.AreEqual(4, table.Columns.Count);
Assert.AreEqual("date", table.Columns[0].ColumnName);
Assert.AreEqual("open", table.Columns[1].ColumnName);
Assert.AreEqual("close", table.Columns[2].ColumnName);
Assert.AreEqual("volume", table.Columns[3].ColumnName);
Assert.AreEqual(GetDate("2008-07-07"), (DateTime)table.Rows[0]["date"]);
Assert.AreEqual(38.37d, (double)table.Rows[0]["open"], 0.000001d);
Assert.AreEqual(38.93d, (double)table.Rows[0]["close"], 0.000001d);
Assert.AreEqual(1155400, (int)table.Rows[0]["volume"]);
Assert.AreEqual(GetDate("2008-06-09"), (DateTime)table.Rows[20]["date"]);
Assert.AreEqual(45.21d, (double)table.Rows[20]["open"], 0.000001d);
Assert.AreEqual(44.50d, (double)table.Rows[20]["close"], 0.000001d);
Assert.AreEqual(1113865, (int)table.Rows[20]["volume"]);
}
public void ArivaEps_DE0005140008()
{
var locator = new DatumLocator("Eps",
new Site("Ariva",
new Navigation(DocumentType.Html,
OS.CombinePaths(TestDataRoot, "Recognition", "ariva.fund.${stock.isin}.html")),
new PathSeriesFormat("Ariava.Eps")
{
Path = "/BODY[0]/DIV[5]/DIV[0]/DIV[3]/TABLE[7]/TBODY[0]/TR[5]/TD[1]",
TimeAxisPosition = 1,
Expand = CellDimension.Row,
SeriesNamePosition = 0,
ValueFormat = new FormatColumn("value", typeof(double), "000000,0000"),
TimeAxisFormat = new FormatColumn("year", typeof(int), "0000")
},
new DataContent("Euro")));
var fetchPolicy = new LookupPolicy();
fetchPolicy.Lut["${stock.isin}"] = "DE0005140008";
var webScrapSC = new ServiceProvider().Browser();
var provider = new GenericDatumProvider(webScrapSC, locator, fetchPolicy, null);
var result = provider.Fetch();
Assert.IsNotNull(result);
Assert.IsNotNull(result.ResultTable);
var table = result.ResultTable;
Assert.AreEqual(6, table.Rows.Count);
Assert.AreEqual(2002, (int)table.Rows[0][1]);
Assert.AreEqual(0.64d, (double)table.Rows[0][0], 0.00001d);
Assert.AreEqual(2007, (int)table.Rows[5][1]);
Assert.AreEqual(13.65d, (double)table.Rows[5][0], 0.00001d);
}
public IDatumProvider Create(DatumLocator datumLocator)
{
return(new GenericDatumProvider(myWebScrapSC, datumLocator, FetchPolicy, ResultPolicy));
}
private static IDatumProvider CreateProvider(DatumLocator datum, IFetchPolicy fetchPolicy)
{
var factory = new DatumProviderFactory(Engine.ServiceProvider.Browser(), fetchPolicy);
return(factory.Create(datum));
}
public static DataTable Fetch(this DatumLocator datum, StockHandle stock)
{
var provider = CreateProvider(datum, new StockLookupPolicy(stock));
return(provider.Fetch().ResultTable);
}
public static SingleResultValue <T> FetchSingle <T>(this DatumLocator datum, StockHandle stock)
{
var provider = CreateProvider(datum, new StockLookupPolicy(stock));
return(provider.FetchSingle <T>());
}
private void PrepareImport()
{
Interpreter.Context.Scope["TestDataDir"] = TestDataRoot;
Eps.Create();
Dividend.Create();
StockPrice.Create();
var epsProvider = new DatumLocator("eps",
new Site("Ariva",
new Navigation(DocumentType.Html, "${TestDataDir}/ariva.html"),
new PathSeriesFormat("Ariva.Eps")
{
Path = @"/BODY[0]/DIV[5]/DIV[0]/DIV[1]/TABLE[7]/TBODY[0]/TR[6]/TD[1]",
TimeAxisPosition = 1,
Expand = CellDimension.Row,
SeriesNamePosition = 0,
// seriesname-contains="unverwässertes Ergebnis pro Aktie">
ValueFormat = new FormatColumn("value", typeof(double), Format.PriceDE),
TimeAxisFormat = new FormatColumn("year", typeof(int), Format.Number)
},
new DataContent("Euro")));
Interpreter.Context.DatumProviderFactory.LocatorRepository.Add(epsProvider);
var dividendProvider = new DatumLocator("dividend",
new Site("Sheet",
new Navigation(DocumentType.Text, @"${TestDataDir}/dividend.csv"),
new SeparatorSeriesFormat("Sheet.Dividend")
{
Separator = ";",
Anchor = Anchor.ForRow(new StringContainsLocator(1, "${stock.isin}")),
TimeAxisPosition = 0,
Expand = CellDimension.Row,
SeriesNamePosition = 1,
SkipColumns = new int[] { 0, 2, 3 },
SkipRows = new[] { 1 },
ValueFormat = new FormatColumn("value", typeof(double), Format.PriceDE),
TimeAxisFormat = new FormatColumn("year", typeof(int), Format.Number)
},
new DataContent("Euro")));
Interpreter.Context.DatumProviderFactory.LocatorRepository.Add(dividendProvider);
var stockPriceProvider = new DatumLocator("stock_price",
new Site("TaiPan",
new Navigation(DocumentType.Text, @"${TestDataDir}/*/555200.TXT"),
new CsvFormat("TaiPan.Prices", ";",
new FormatColumn("date", typeof(DateTime), "dd.MM.yyyy"),
new FormatColumn("close", typeof(double), Format.PriceDE),
new FormatColumn("volume", typeof(int), Format.PriceDE),
new FormatColumn("high", typeof(double), Format.PriceDE),
new FormatColumn("low", typeof(double), Format.PriceDE),
new FormatColumn("open", typeof(double), Format.PriceDE)),
new DataContent("Euro")));
Interpreter.Context.DatumProviderFactory.LocatorRepository.Add(stockPriceProvider);
using (var tom = Engine.ServiceProvider.CreateEntityRepository())
{
var cur = new Currency("Euro", "Euro");
tom.Currencies.AddObject(cur);
tom.SaveChanges();
}
}