public async Task <List <MonthlyCategoryExpense> > GetMonthCategoryExpenses(MonthlyCategory category, Uri paginatedLink = null, bool autoPaginate = true)
{
// @todo compare category agency, year, month against last search value and auto-search again if needed
using (var http = new HttpClient(_handler, false))
{
var url = (paginatedLink != null) ? paginatedLink : category.CategoryUrl;
var response = await http.GetStringAsync(url);
var doc = new HtmlDocument();
doc.LoadHtml(response);
_lastWebFormsValues = ParseWebForms(doc);
var paginationNode = doc.DocumentNode.SelectSingleNode("//p[ contains( @class, 'paginationNote' ) ]");
var paginationRegex = new Regex(@"Displaying records (\d+) through (\d+) of (\d+) found.");
var nextNode =
doc.DocumentNode.SelectSingleNode("//div[ contains( @class, 'nextBack' ) ]/a[ contains( text(), 'Next' ) ]");
var expenseRows =
doc.DocumentNode.SelectNodes(
"//table[ @id='ctl00_ContentPlaceHolder_ObjectDataControl_CategoryDataTable' ]//tr[ position() > 1 ]");
var expenses = new List <MonthlyCategoryExpense>();
foreach (var expenseRow in expenseRows)
{
var linkNode = expenseRow.SelectSingleNode(".//a");
var expenseName = linkNode.InnerText;
var expenseLink = linkNode.GetAttributeValue("href", "");
var amount = expenseRow.SelectSingleNode("./td[ contains( @class, 'total_highlight' ) ]").InnerText;
var expense = new MonthlyCategoryExpense()
{
Expense = expenseName,
ExpenseUrl = new Uri(new Uri("https://applications.sc.gov/SpendingTransparency/"), expenseLink),
Amount = Convert.ToDecimal(amount.Replace("$", "").Replace(",", "")),
Agency = category.Agency,
Year = category.Year,
Month = category.Month,
Category = category
};
expenses.Add(expense);
}
// if there's another page of expenses, add those recursively - only if we haven't disabled paginating
if (nextNode != null && autoPaginate)
{
var nextLink = new Uri(new Uri("https://applications.sc.gov/SpendingTransparency/"),
nextNode.GetAttributeValue("href", ""));
expenses.AddRange(await GetMonthCategoryExpenses(category, nextLink));
}
return(expenses);
}
}
public async Task <List <MonthlyCategory> > GetMonthCategories(Agency agency, Year year, Month month)
{
// we actually only need to get new webforms values if we've done something other than searching for categories before, but we'll do it always
// since we've usually continued on to do something else at this point -- all this does is populate the last webforms values with the right info
await GetSearchValues();
using (var http = new HttpClient(_handler, false))
{
var postBody = new FormUrlEncodedContent(AddWebForms(new List <KeyValuePair <string, string> >()
{
new KeyValuePair <string, string>("ctl00$ContentPlaceHolder$SearchControl$YearDropdownList", year.SearchValue),
new KeyValuePair <string, string>("ctl00$ContentPlaceHolder$SearchControl$MonthDropdownList", month.SearchValue),
new KeyValuePair <string, string>("ctl00$ContentPlaceHolder$SearchControl$AgencyDropdownList", agency.SearchValue),
new KeyValuePair <string, string>("ctl00$ContentPlaceHolder$SearchControl$SearchButton", "Search"),
}));
var response =
await http.PostAsync("https://applications.sc.gov/SpendingTransparency/MonthlyExpenditureSearch.aspx?etype=1",
postBody);
// make sure the request was successful before we continue
response.EnsureSuccessStatusCode();
var responseBody = await response.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(responseBody);
// get all the table rows, excluding the first, which contains the headers
var tableRows =
doc.DocumentNode.SelectNodes(
"//table[ @id='ctl00_ContentPlaceHolder_CategoryDataControl_CategoryDataTable' ]//tr[ position() > 1 ]");
if (tableRows == null)
{
return(new List <MonthlyCategory>());
}
var categories = new List <MonthlyCategory>();
foreach (var tableRow in tableRows)
{
var linkNode = tableRow.SelectSingleNode(".//a");
var categoryName = linkNode.InnerText;
var categoryLink = linkNode.GetAttributeValue("href", "");
var amount = tableRow.SelectSingleNode("./td[ contains( @class, 'total_highlight' ) ]").InnerText;
var category = new MonthlyCategory()
{
Category = categoryName,
CategoryUrl = new Uri(new Uri("https://applications.sc.gov/SpendingTransparency/"), categoryLink),
Amount = Convert.ToDecimal(amount.Replace("$", "").Replace(",", "")),
Agency = agency,
Year = year,
Month = month
};
categories.Add(category);
}
return(categories);
}
}