private async Task ScrapeStockTable(Page page, ResultCollection <ResultItemCollection> resultCollection, Dictionary <string, object> args = null)
{
var resultSelector = "#MAIN_BODY";
await page.WaitForSelectorAsync(resultSelector);
//try to get the table headers
var mappings = _scrappedSettingRepository.GetItemSettings("TableDataSetting");
var dataSelector = "#MAIN_BODY table > tbody > tr";
var tData = await PuppeteerHelper.GetTableData(page, dataSelector, mappings);
var identifier = mappings.FirstOrDefault(x => x.IsIdentifier)?.Key;
foreach (JToken link in tData)
{
resultCollection.Add(new ResultItemCollection
{
Key = identifier,
Items = mappings.Select(x => new ResultItem
{
Name = x.Key,
Value = link[x.Key].GetValue()
}).ToList()
});
}
}
private async Task ScrapeHistoricalData(Page page, ResultCollection <ResultItemCollection> resultCollection, Dictionary <string, object> args = null)
{
var resultSelector = "#HistoricalDataTable";
await page.WaitForSelectorAsync(resultSelector);
//try to get the table headers
var mappings = _scrappedSettingRepository.GetItemSettings("HistoricalData");
var dataSelector = "#HistoricalDataTable > tbody > tr";
var tData = await PuppeteerHelper.GetTableData(page, dataSelector, mappings);
var identifier = args["StockCode"].ToString();
var collection = resultCollection.GetItem(identifier);
var date = "";
foreach (var item in tData)
{
if (date != item["Date"]?.GetValue())
{
date = item["Date"].GetValue();
}
collection.Items.AddRange(mappings.Select(x => new ResultItem
{
Name = x.Key,
Value = item[x.Key].GetValue(),
Group = date,
IsMultiple = true
}).ToList());
}
}
private async Task ScrapeRealTimeMonitoringData(Page page, ResultCollection <ResultItemCollection> resultCollection, Dictionary <string, object> args)
{
var resultSelector = "#StockQuoteTable";
await page.WaitForSelectorAsync(resultSelector);
//try to get the table headers
var mappings = _scrappedSettingRepository.GetItemSettings("RealTimeMonitoring");
var dataSelector = "#StockQuoteTable > tbody > tr";
var tData = await PuppeteerHelper.GetTableData(page, dataSelector, mappings);
foreach (JToken link in tData)
{
resultCollection.Add(new ResultItemCollection
{
Key = link["StockCode"].GetValue(),
Items = mappings.Select(x => new ResultItem
{
Name = x.Key,
Value = link[x.Key].GetValue()
}).ToList()
});
}
}
