public async Task DownloadAsync(string downloadUrl, string sdf)
{
string downloadLocation = "";
using (var client = new BetterWebClient(this.cookieContainer, false))
{
string downloadPage = await client.DownloadStringTaskAsync(downloadUrl);
CQ downloadPageDom = CQ.Create(downloadPage);
var downloadInputButton = downloadPageDom.Find("#optionsPage input");
downloadLocation = downloadInputButton.Attr("onClick");
// e.g. window.location='https://earthexplorer.usgs.gov/download/4220/ASTGDEMV2_0N51W004/STANDARD/INVSVC'
downloadLocation = downloadLocation.Replace("window.location", "");
downloadLocation = downloadLocation.Replace("=", "");
downloadLocation = downloadLocation.Trim('\'');
}
if (!String.IsNullOrEmpty(downloadLocation))
{
using (var client = new BetterWebClient(this.cookieContainer, false))
{
string nextPage = await client.DownloadStringTaskAsync(downloadLocation);
CQ nextPageDom = CQ.Create(nextPage);
string nextPageLower = nextPage.ToLower();
if (nextPageLower.Contains("agree to") && nextPageLower.Contains("end user license"))
{
await this.AcceptDownloadLicenseAgreement(client, downloadLocation, nextPageDom);
}
}
}
}
public async Task <string> GetContentAsync(string url)
{
var res = await client.DownloadStringTaskAsync(new Uri(url));
LastCookie = this.client.CookieContainer.GetCookieHeader(new Uri(url));
return(res);
}
public async Task LoginAsync(string username, string password)
{
// As of 2018-07-05 the following is posted to login
// __ncforminfo:{from hidden input}
// csrf_token:{from hidden input}
// password:******
// username:username
var loginUrl = "https://ers.cr.usgs.gov/login/";
this.cookieContainer = new CookieContainer();
using (var client = new BetterWebClient(this.cookieContainer, false))
{
string data = await client.DownloadStringTaskAsync(loginUrl);
CQ dom = CQ.Create(data);
var csrfValue = dom["#csrf_token"].Val();
var ncFormInfo = dom["input[name='__ncforminfo']"].Val();
var postParams = new NameValueCollection();
postParams.Add("__ncforminfo", ncFormInfo);
postParams.Add("csrf_token", csrfValue);
postParams.Add("username", username);
postParams.Add("password", password);
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
client.Headers.Add("Accept-Encoding", "gzip, deflate");
client.Headers.Add("Accept-Language", "en-US,en;q=0.5");
client.Headers.Add("Cache-Control", "no-cache");
client.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
client.Headers.Add("Host", "ers.cr.usgs.gov");
client.Headers.Add("Pragma", "no-cache");
client.Headers.Add("Referer", loginUrl);
client.Headers.Add("User-Agent", AeroSceneryManager.Instance.Settings.UserAgent);
var responseBytes = await client.UploadValuesTaskAsync(loginUrl, "POST", postParams);
string responsebody = Encoding.UTF8.GetString(responseBytes);
var locationHeader = client.ResponseHeaders["Location"].ToString();
//string data2 = await client.DownloadStringTaskAsync("https://earthexplorer.usgs.gov/");
int i = 0;
}
}
public async Task <IList <FSCloudPortAirport> > ScrapeAirportsAsync()
{
var urlTemplate = "http://www.fscloudport.com/phdi/p1.nsf/aeroscenery?OpenView&Start={0}&Count={1}";
Dictionary <string, FSCloudPortAirport> airports = new Dictionary <string, FSCloudPortAirport>();
try
{
using (var client = new BetterWebClient(null, false))
{
bool dataAvailable = true;
int startIndex = 1;
int count = 1000;
do
{
string url = String.Format(urlTemplate, startIndex, count);
string page = await client.DownloadStringTaskAsync(url);
if (page.Contains("No documents found"))
{
dataAvailable = false;
}
ParseAirportListHtml(page, airports);
startIndex += 1000;
}while (dataAvailable);
}
}
catch (System.Net.WebException ex)
{
log.Error("Error scraping FSCloudPort airports", ex);
}
return(airports.Values.ToList());
}