private List <ExtractedValues> FetchAllPoliticians()
{
List <ExtractedValues> politicians = new List <ExtractedValues>();
bool hasNextLink = true;
while (hasNextLink)
{
OdataClient client = new OdataClient(url);
Odata page = client.fetchData();
foreach (var politician in page.Value)
{
BiographyReader reader = new BiographyReader(politician.Biografi);
reader.ReadBiography();
ExtractedValues tempPol = reader.GetPolitician();
politicians.Add(tempPol);
}
if (page.NextLink == null)
{
hasNextLink = false;
}
else
{
url = page.NextLink;
}
}
return(politicians);
}
private ExtractedValues CombineToPolitician(string firstname, string lastname, string party, string formattedTitle, string partyShortname, ContactInfo contact)
{
politicianValues = new ExtractedValues {
Firstname = firstname,
Lastname = lastname, Party = party,
Title = formattedTitle,
PartyShortname = partyShortname,
Contact = contact
};
return(politicianValues);
}
private ExtractedValues ConvertToPolitician(string line, string contactLine)
{
string[] splittedLine;
string[] splittedContactLine;
splittedLine = line.Split(' ');
splittedContactLine = contactLine.Split(' ');
ExtractedValues politician = new ExtractedValues();
var name = GetNameFromSL(splittedLine);
politician.Title = GetTitleFromSL(splittedLine);
politician.Firstname = name.Item1;
politician.Lastname = name.Item2;
politician.Party = GetPartyFromSL(splittedLine);
politician.Contact = GetContactFromCL(splittedContactLine);
return(politician);
}
public async Task <List <BalanceSheet> > ScrapeBalanceSheet(string ticker, string apiKey)
{
#region Data from QuickFS
var url = $" https://api.quickfs.net/stocks/{ticker}:US/bs/Annual/{apiKey}";
var helper = new WebHelper();
var request = await helper.ComposeWebRequestGet(url);
var result = await helper.CallWebRequest(request);
result = result.Replace("<\\/td>", "");
var html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(result);
var htmlNodes = html.DocumentNode.Descendants("td").ToList();
#endregion
#region DataOrganization
#region WordRemover
var word = "Liabilities & Equity";
var isDeleted = false;
foreach (var nodeItem in htmlNodes.ToList())
{
if (nodeItem.InnerHtml.Equals(word) && isDeleted == false)
{
nodeItem.InnerHtml = "";
isDeleted = true;
}
}
#endregion
var numberOfColumns = html.DocumentNode.SelectNodes("//tr[@class='thead']").Descendants("td").ToList().Count();
var numberOfRows = htmlNodes.Count / numberOfColumns;
var count = 1;
var extractedValuesList = new List <ExtractedValues>();
var valuesFinalList = new List <float>();
var balanceSheets = new List <BalanceSheet>();
for (var i = 1; i < numberOfColumns; i++)
{
var parsedYear = int.TryParse(htmlNodes[i].InnerText, out int yearNumber);
//if (!parsedYear) return; lançar exceção
for (var j = 1; j < numberOfRows; j++)
{
var extractedValues = new ExtractedValues();
var baseItems = new BaseItem();
var name = htmlNodes[j * numberOfColumns].InnerText;
baseItems.Name = name;
var valuesList = new List <string>();
foreach (var item in htmlNodes)
{
var htmlValue = item.GetAttributeValue("data-value", null);
valuesList.Add(htmlValue);
}
var valuesFromNodes = valuesList[(j * numberOfColumns) + count];
bool parsedFloat = float.TryParse(valuesFromNodes, NumberStyles.Float, CultureInfo.InvariantCulture, out float valuesFloat);
valuesFinalList.Add(valuesFloat);
if (yearNumber != 0 && name != "" && name != "Assets")
{
extractedValues.Year = yearNumber;
baseItems.Name = name;
baseItems.Value = valuesFloat;
extractedValues.Items.Add(baseItems);
extractedValuesList.Add(extractedValues);
}
}
#endregion
#region Add to BalanceSheet
var balanceSheet = new BalanceSheet();
foreach (var extractedItem in extractedValuesList)
{
var props = balanceSheet.GetType().GetProperties();
balanceSheet.Year = extractedItem.Year;
foreach (var prop in props)
{
var displayAttribute = prop.GetCustomAttributes <DisplayAttribute>().SingleOrDefault();
if (displayAttribute != null)
{
var item = extractedItem.Items.SingleOrDefault(i => i.Name == displayAttribute.Name);
if (item != null)
{
prop.SetValue(balanceSheet, item.Value);
}
}
}
}
if (balanceSheet.Year != 0)
{
balanceSheets.Add(balanceSheet);
}
#endregion
}
Random rnd = new Random();
await Task.Delay(TimeSpan.FromSeconds(rnd.Next(1, 10)));
return(balanceSheets);
}
public async Task <List <CashFlowStatement> > ScrapeCashFlowStatement(string ticker, string apiKey)
{
#region Data From QuickFS
string url = "https://api.quickfs.net/stocks/" + ticker + ":US/cf/Annual/" + apiKey;
var helper = new WebHelper();
var request = await helper.ComposeWebRequestGet(url);
var result = await helper.CallWebRequest(request);
result = result.Replace("<\\/td>", "");
result = result.Replace("$", "");
result = result.Replace(",", ".");
var html = new HtmlAgilityPack.HtmlDocument();
html.LoadHtml(result);
var htmlNodes = html.DocumentNode.Descendants("td").ToList();
var numberOfColumns = html.DocumentNode.SelectNodes("//tr[@class='thead']").Descendants("td").ToList().Count();
var numberOfRows = htmlNodes.Count / numberOfColumns;
var count = 1;
#endregion
#region Data Organization
var namesList = new List <string>();
var cashFlowStatements = new List <CashFlowStatement>();
var cashFlowStatementForCompanyList = new List <CashFlowStatement>();
for (var i = 1; i < numberOfColumns; i++)
{
var extractedValuesList = new List <ExtractedValues>();
for (var h = 1; h < numberOfRows; h++)
{
var name = htmlNodes[h * numberOfColumns].InnerText;
if (name == "Other")
{
name = name + h;
}
namesList.Add(name);
}
var parsedYear = int.TryParse(htmlNodes[i].InnerText, out int yearNumber);
//if (!parsedYear) return; lançar exceção
for (var j = 1; j < numberOfRows; j++)
{
var extractedValues = new ExtractedValues();
var baseItems = new BaseItem();
var name = namesList[j - 1];
baseItems.Name = name;
var valuesList = new List <string>();
foreach (var item in htmlNodes)
{
var htmlValue = item.GetAttributeValue("data-value", null);
valuesList.Add(htmlValue);
}
var valuesFromNodes = valuesList[(j * numberOfColumns) + count];
bool parsedFloat = float.TryParse(valuesFromNodes, NumberStyles.Float, CultureInfo.InvariantCulture, out float valuesFloat);
if (yearNumber != 0 && name != "" && name != "Operating Expenses")
{
extractedValues.Year = yearNumber;
baseItems.Name = name;
baseItems.Value = valuesFloat;
extractedValues.Items.Add(baseItems);
extractedValuesList.Add(extractedValues);
}
}
count++;
#endregion
#region Add to CashFlowStatement
foreach (var extractedItem in extractedValuesList)
{
var cashFlowStatement = new CashFlowStatement();
var props = cashFlowStatement.GetType().GetProperties();
cashFlowStatement.Year = extractedItem.Year;
foreach (var prop in props)
{
var displayAttribute = prop.GetCustomAttributes <DisplayAttribute>().SingleOrDefault();
if (displayAttribute != null)
{
var item = extractedItem.Items.SingleOrDefault(i => i.Name == displayAttribute.Name);
if (item != null)
{
prop.SetValue(cashFlowStatement, item.Value);
}
}
}
cashFlowStatements.Add(cashFlowStatement);
}
//if (cashFlowStatement.Year != 0) cashFlowStatementForCompanyList.Add(cashFlowStatement);
#endregion
}
await Task.Delay(TimeSpan.FromSeconds(2));
return(cashFlowStatements);
}
public async Task <List <KeyRatio> > ScrapeKeyRatio(string ticker, string apiKey)
{
#region Data from QuickFS
string url = "https://api.quickfs.net/stocks/" + ticker + ":US/ratios/Annual/" + apiKey;
var helper = new WebHelper();
var request = await helper.ComposeWebRequestGet(url);
var result = await helper.CallWebRequest(request);
result = result.Replace("<\\/td>", "");
result = result.Replace("<\\/tr>", "");
result = result.Replace("$", "");
result = result.Replace(",", ".");
var html = new HtmlDocument();
html.LoadHtml(result);
var htmlNodes = html.DocumentNode.Descendants("td").ToList();
var numberOfColumns = html.DocumentNode.SelectNodes("//tr[@class='thead']").Descendants("td").ToList().Count();
var numberOfRows = htmlNodes.Count / numberOfColumns;
var count = 1;
#endregion
#region DataOrganization
var namesList = new List <string>();
var keyRatios = new List <KeyRatio>();
var valuesFinalList = new List <float>();
for (var i = 1; i < numberOfColumns; i++)
{
var extractedValuesList = new List <ExtractedValues>();
for (var h = 1; h < numberOfRows; h++)
{
var name = htmlNodes[h * numberOfColumns].InnerText;
if (name == "Revenue" || name == "EBITDA" || name == "Operating Income" || name == "Free Cash Flow" || name == "Book Value" || name == "Tangible Book Value")
{
name = name + h;
}
namesList.Add(name);
}
var parsedYear = int.TryParse(htmlNodes[i].InnerText, out int yearNumber);
//if (!parsedYear) return; lançar exceção
for (var j = 1; j < numberOfRows; j++)
{
var name = htmlNodes[j * numberOfColumns].InnerText;
var valuesList = new List <string>();
foreach (var item in htmlNodes)
{
var htmlValue = item.GetAttributeValue("data-value", null);
valuesList.Add(htmlValue);
}
var valuesFromNodes = valuesList[(j * numberOfColumns) + count];
bool parsedFloat = float.TryParse(valuesFromNodes, NumberStyles.Float, CultureInfo.InvariantCulture, out float valuesFloat);
valuesFinalList.Add(valuesFloat);
if (yearNumber != 0 && name != "" && name != "Returns")
{
var baseItems = new BaseItem();
var extractedValues = new ExtractedValues();
extractedValues.Year = yearNumber;
baseItems.Name = name;
baseItems.Value = valuesFloat;
extractedValues.Items.Add(baseItems);
extractedValuesList.Add(extractedValues);
}
}
count++;
#endregion
#region Add to KeyRatio
foreach (var extractedItem in extractedValuesList)
{
var keyRatio = new KeyRatio();
var props = keyRatio.GetType().GetProperties();
keyRatio.Year = extractedItem.Year;
foreach (var prop in props)
{
var displayAttribute = prop.GetCustomAttributes <DisplayAttribute>().SingleOrDefault();
if (displayAttribute != null)
{
var item = extractedItem.Items.SingleOrDefault(i => i.Name == displayAttribute.Name);
if (item != null)
{
prop.SetValue(keyRatio, item.Value);
}
}
}
if (keyRatio.Year != 0)
{
keyRatios.Add(keyRatio);
}
}
#endregion
}
Random rnd = new Random();
await Task.Delay(TimeSpan.FromSeconds(rnd.Next(1, 10)));
return(keyRatios);
}
