public override void Init() { this.LoggingLevel = WebScraper.LogLevel.All; this.HttpRetryAttempts = 1; this.HttpTimeOut = TimeSpan.FromSeconds(3); var identity = new HttpIdentity(); identity.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0"; identity.HttpRequestHeaders.Add("Accept", "*/*"); identity.HttpRequestHeaders.Add("Accept-Language", "en-US,en;q=0.5"); identity.HttpRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); identity.HttpRequestHeaders.Add("Accept-Charset", "ISO - 8859 - 1, utf - 8; q = 0.7,*; q = 0.3"); identity.HttpRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0"); identity.HttpRequestHeaders.Add("Host", "developer.mozilla.org"); identity.HttpRequestHeaders.Add("Content-Type", "application/x-www-form-urlencoded"); identity.UseCookies = true; this.Identities.Add(identity); }
public override void Init() { this.LoggingLevel = WebScraper.LogLevel.All; this.HttpRetryAttempts = 1; this.HttpTimeOut = TimeSpan.FromSeconds(3); var identity = new HttpIdentity(); identity.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0"; identity.HttpRequestHeaders.Add("Accept", "*/*"); identity.HttpRequestHeaders.Add("Accept-Language", "en-US,en;q=0.5"); identity.HttpRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); identity.HttpRequestHeaders.Add("Accept-Charset", "ISO - 8859 - 1, utf - 8; q = 0.7,*; q = 0.3"); identity.HttpRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:50.0) Gecko/20100101 Firefox/50.0"); identity.HttpRequestHeaders.Add("Host", "developer.mozilla.org"); identity.HttpRequestHeaders.Add("Content-Type", "application/x-www-form-urlencoded"); identity.UseCookies = true; this.Identities.Add(identity); this.Request("http://machinerymax.com/Event/Details/10145158/COMING-SOON-Cornerstone-Architectural-Concepts-COMPLETE-SHOP-CLOSURE", Parse, identity); }
public override void Init() { License.LicenseKey = "LicenseKey"; // Write License Key this.LoggingLevel = WebScraper.LogLevel.All; // All Events Are Logged HttpIdentity identity = new HttpIdentity(); identity.HttpRequestHeaders.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; " + "Windows NT 5.2; .NET CLR 1.0.3705;)"); foreach (var UA in IronWebScraper.CommonUserAgents.ChromeDesktopUserAgents) { Identities.Add(new HttpIdentity() { UserAgent = UA, UseCookies = true }); } this.Request(this._url, Parse); }
public override void Init() { this.LoggingLevel = WebScraper.LogLevel.All; HttpIdentity foo = new HttpIdentity(); foo.HttpRequestHeaders = new Dictionary <string, string>() { { "authority", "www.zillow.com" }, { "method", "GET" }, { "path", "/homedetails/20-Plumwood-Rd-Dayton-OH-45409/35082326_zpid/?fullpage=true" }, { "scheme", "https" }, { "accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" }, { "accept-encoding", "gzip, deflate, br" }, { "accept-language", "en-US,en;q=0.9" }, { "cache-control", "max-age=0" }, { "referer", "https://www.zillow.com/homedetails/20-Plumwood-Rd-Dayton-OH-45409/35082326_zpid/" }, { "upgrade-insecure-requests", "1" }, { "user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36" }, }; this.Request("https://www.zillow.com/homedetails/20-Plumwood-Rd-Dayton-OH-45409/35082326_zpid/", Parse, identity: foo); }