public async Task DownloadAsync(string downloadUrl, string sdf) { string downloadLocation = ""; using (var client = new BetterWebClient(this.cookieContainer, false)) { string downloadPage = await client.DownloadStringTaskAsync(downloadUrl); CQ downloadPageDom = CQ.Create(downloadPage); var downloadInputButton = downloadPageDom.Find("#optionsPage input"); downloadLocation = downloadInputButton.Attr("onClick"); // e.g. window.location='https://earthexplorer.usgs.gov/download/4220/ASTGDEMV2_0N51W004/STANDARD/INVSVC' downloadLocation = downloadLocation.Replace("window.location", ""); downloadLocation = downloadLocation.Replace("=", ""); downloadLocation = downloadLocation.Trim('\''); } if (!String.IsNullOrEmpty(downloadLocation)) { using (var client = new BetterWebClient(this.cookieContainer, false)) { string nextPage = await client.DownloadStringTaskAsync(downloadLocation); CQ nextPageDom = CQ.Create(nextPage); string nextPageLower = nextPage.ToLower(); if (nextPageLower.Contains("agree to") && nextPageLower.Contains("end user license")) { await this.AcceptDownloadLicenseAgreement(client, downloadLocation, nextPageDom); } } } }
public async Task <string> GetContentAsync(string url) { var res = await client.DownloadStringTaskAsync(new Uri(url)); LastCookie = this.client.CookieContainer.GetCookieHeader(new Uri(url)); return(res); }
public async Task LoginAsync(string username, string password) { // As of 2018-07-05 the following is posted to login // __ncforminfo:{from hidden input} // csrf_token:{from hidden input} // password:****** // username:username var loginUrl = "https://ers.cr.usgs.gov/login/"; this.cookieContainer = new CookieContainer(); using (var client = new BetterWebClient(this.cookieContainer, false)) { string data = await client.DownloadStringTaskAsync(loginUrl); CQ dom = CQ.Create(data); var csrfValue = dom["#csrf_token"].Val(); var ncFormInfo = dom["input[name='__ncforminfo']"].Val(); var postParams = new NameValueCollection(); postParams.Add("__ncforminfo", ncFormInfo); postParams.Add("csrf_token", csrfValue); postParams.Add("username", username); postParams.Add("password", password); client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); client.Headers.Add("Accept-Encoding", "gzip, deflate"); client.Headers.Add("Accept-Language", "en-US,en;q=0.5"); client.Headers.Add("Cache-Control", "no-cache"); client.Headers.Add("Content-Type", "application/x-www-form-urlencoded"); client.Headers.Add("Host", "ers.cr.usgs.gov"); client.Headers.Add("Pragma", "no-cache"); client.Headers.Add("Referer", loginUrl); client.Headers.Add("User-Agent", AeroSceneryManager.Instance.Settings.UserAgent); var responseBytes = await client.UploadValuesTaskAsync(loginUrl, "POST", postParams); string responsebody = Encoding.UTF8.GetString(responseBytes); var locationHeader = client.ResponseHeaders["Location"].ToString(); //string data2 = await client.DownloadStringTaskAsync("https://earthexplorer.usgs.gov/"); int i = 0; } }
public async Task <IList <FSCloudPortAirport> > ScrapeAirportsAsync() { var urlTemplate = "http://www.fscloudport.com/phdi/p1.nsf/aeroscenery?OpenView&Start={0}&Count={1}"; Dictionary <string, FSCloudPortAirport> airports = new Dictionary <string, FSCloudPortAirport>(); try { using (var client = new BetterWebClient(null, false)) { bool dataAvailable = true; int startIndex = 1; int count = 1000; do { string url = String.Format(urlTemplate, startIndex, count); string page = await client.DownloadStringTaskAsync(url); if (page.Contains("No documents found")) { dataAvailable = false; } ParseAirportListHtml(page, airports); startIndex += 1000; }while (dataAvailable); } } catch (System.Net.WebException ex) { log.Error("Error scraping FSCloudPort airports", ex); } return(airports.Values.ToList()); }