private async Task ScrapeStockTable(Page page, ResultCollection <ResultItemCollection> resultCollection, Dictionary <string, object> args = null) { var resultSelector = "#MAIN_BODY"; await page.WaitForSelectorAsync(resultSelector); //try to get the table headers var mappings = _scrappedSettingRepository.GetItemSettings("TableDataSetting"); var dataSelector = "#MAIN_BODY table > tbody > tr"; var tData = await PuppeteerHelper.GetTableData(page, dataSelector, mappings); var identifier = mappings.FirstOrDefault(x => x.IsIdentifier)?.Key; foreach (JToken link in tData) { resultCollection.Add(new ResultItemCollection { Key = identifier, Items = mappings.Select(x => new ResultItem { Name = x.Key, Value = link[x.Key].GetValue() }).ToList() }); } }
private async Task ScrapeHistoricalData(Page page, ResultCollection <ResultItemCollection> resultCollection, Dictionary <string, object> args = null) { var resultSelector = "#HistoricalDataTable"; await page.WaitForSelectorAsync(resultSelector); //try to get the table headers var mappings = _scrappedSettingRepository.GetItemSettings("HistoricalData"); var dataSelector = "#HistoricalDataTable > tbody > tr"; var tData = await PuppeteerHelper.GetTableData(page, dataSelector, mappings); var identifier = args["StockCode"].ToString(); var collection = resultCollection.GetItem(identifier); var date = ""; foreach (var item in tData) { if (date != item["Date"]?.GetValue()) { date = item["Date"].GetValue(); } collection.Items.AddRange(mappings.Select(x => new ResultItem { Name = x.Key, Value = item[x.Key].GetValue(), Group = date, IsMultiple = true }).ToList()); } }
private async Task ScrapeRealTimeMonitoringData(Page page, ResultCollection <ResultItemCollection> resultCollection, Dictionary <string, object> args) { var resultSelector = "#StockQuoteTable"; await page.WaitForSelectorAsync(resultSelector); //try to get the table headers var mappings = _scrappedSettingRepository.GetItemSettings("RealTimeMonitoring"); var dataSelector = "#StockQuoteTable > tbody > tr"; var tData = await PuppeteerHelper.GetTableData(page, dataSelector, mappings); foreach (JToken link in tData) { resultCollection.Add(new ResultItemCollection { Key = link["StockCode"].GetValue(), Items = mappings.Select(x => new ResultItem { Name = x.Key, Value = link[x.Key].GetValue() }).ToList() }); } }