public async Task AsyncHttpGetRequest() { if (Helper.IsNetworkAvailable()) { var http = new DefaultHttpRequester(); var request = new Request { Address = new Url("http://httpbin.org/robots.txt"), Method = HttpMethod.Get }; using (var response = http.RequestAsync(request, CancellationToken.None)) { Assert.IsNotNull(response); Assert.IsFalse(response.IsCompleted); var result = await response; Assert.IsTrue(response.IsCompleted); Assert.IsTrue(result.Content.CanRead); Assert.IsTrue(result.Headers.Count > 0); var content = new StreamReader(result.Content); Assert.AreEqual("User-agent: *\nDisallow: /deny\n", content.ReadToEnd()); } } }
public AngleSharpDriverRequester() { var configuration = GetBasicConfiguration(); defaultHttpRequester = new DefaultHttpRequester(); this.client = BrowsingContext.New(configuration); }
private async Task CreateContext() { if (context != null) { return; } Console.WriteLine("Notabenoid login"); var requester = new DefaultHttpRequester(); requester.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0"; context = BrowsingContext.New(Configuration.Default.With(requester).WithDefaultLoader().WithDefaultCookies()); // Login { var queryDocument = await GetDocumentAsync("http://notabenoid.org/"); var form = queryDocument.QuerySelector <IHtmlFormElement>("form"); await form.SubmitAsync(new Dictionary <string, string> { { "login[login]", _notabenoidLogin }, { "login[pass]", _notabenoidPassword }, }); } }
public async Task RequestUserAgentString() { if (Helper.IsNetworkAvailable()) { var agent = "MyAgent"; var http = new DefaultHttpRequester(agent); var request = new Request { Address = new Url("http://httpbin.org/user-agent"), Method = HttpMethod.Get }; using (var response = await http.RequestAsync(request, CancellationToken.None)) { Assert.IsNotNull(response); Assert.AreEqual(200, (int)response.StatusCode); Assert.IsTrue(response.Content.CanRead); Assert.IsTrue(response.Headers.Count > 0); var stream = new StreamReader(response.Content); Assert.IsNotNull(stream); var content = stream.ReadToEnd(); Assert.IsTrue(content.Length > 0); Assert.AreEqual("{\n \"user-agent\": \"" + agent + "\"\n}\n", content); } } }
public async Task SimpleHttpPutRequest() { if (Helper.IsNetworkAvailable()) { var http = new DefaultHttpRequester(); var request = new Request { Address = new Url("http://httpbin.org/put"), Method = HttpMethod.Put, Content = Helper.StreamFromString("PUT THIS THING BACK") }; using (var response = await http.RequestAsync(request, CancellationToken.None)) { Assert.IsNotNull(response); Assert.AreEqual(200, (int)response.StatusCode); Assert.IsTrue(response.Content.CanRead); Assert.IsTrue(response.Headers.Count > 0); var stream = new StreamReader(response.Content); Assert.IsNotNull(stream); var content = stream.ReadToEnd(); Assert.IsTrue(content.Length > 0); Assert.IsTrue(content.Contains("\"data\": \"PUT THIS THING BACK\"")); } } }
public void ShouldReturnHttpWebResponseAsync() { //Arrange var defaultHttpRequester = new DefaultHttpRequester(); var webRequest = WebRequest.Create("https://www.google.fr"); //Act var response = defaultHttpRequester.GetResponseAsync(webRequest); //Assert Assert.IsNotNull(response.Result); }
public void TestRetrieveTokenWithWrong() { var requester = new DefaultHttpRequester(); var context = new ApiContext(); context.AddParam("grant_type", "client_credential") .AddParam("appid", "wrong AppId") .AddParam("secret", "wrong AppSecret"); Assert.Throws <ApiInvokeException>(() => { var token = requester.Execute <Token>("/cgi-bin/token", context); }); }
public void ShouldThrowArgumentNullException() { //Arrange var defaultHttpRequester = new DefaultHttpRequester(); //Act var ex = Assert.ThrowsException <ArgumentNullException>(() => { defaultHttpRequester.GetResponse(null); }); //Assert Assert.AreEqual("request", ex.ParamName); }
/// <summary> /// Creates a requester to be used for the HTTP requests. /// </summary> /// <returns></returns> private IRequester CreateRequester() { // Create default requester var requester = new DefaultHttpRequester() { Timeout = TimeSpan.FromSeconds(10), }; // Set headers requester.Headers[HeaderNames.UserAgent] = Consts.USER_AGENT; requester.Headers["dnt"] = "1"; return(requester); }
public void TestRetrieveToken() { var x = new SecretInformationReader(); var requester = new DefaultHttpRequester(); var context = new ApiContext(); context.AddParam("grant_type", "client_credential") .AddParam("appid", x.AppId) .AddParam("secret", x.AppSecret); var token = requester.Execute <Token>("/cgi-bin/token", context); Assert.NotNull(token.access_token); }
private async Task <List <dynamic> > GetPageData(string url, List <dynamic> results) { var requester = new DefaultHttpRequester("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0"); var config = Configuration.Default.With(requester).WithDefaultLoader().WithDefaultCookies(); var context = BrowsingContext.New(config); try { IDocument document; if (url.Contains(AmazonUrl)) { //var productId = url.Substring(url.LastIndexOf("/", StringComparison.Ordinal) + 1); var productId = url[(url.LastIndexOf("/", StringComparison.Ordinal) + 1)..];
private static async Task <Dictionary <string, string> > GetHTMLGoogle(string ticker) { var quoteSummary = new Dictionary <string, string>(); var requester = new DefaultHttpRequester(); requester.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"; var config = Configuration.Default.With(requester).WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync($"https://www.google.com/search?q={ticker}"); var quoteSummaryListItems = document.QuerySelector("span[jsname='vWLAgc']"); var tmp = quoteSummaryListItems.TextContent.Replace(",", ""); quoteSummary.Add("CURRENT PRICE", tmp); return(quoteSummary); }
public void ShouldThrowArgumentNullExceptionAsync() { //Arrange var defaultHttpRequester = new DefaultHttpRequester(); //Act var ex = Assert.ThrowsException <AggregateException>(() => { defaultHttpRequester.GetResponseAsync(null).Wait(); }); //Assert ex.Handle((Exception arg) => { var argException = arg as ArgumentNullException; Assert.IsNotNull(argException); Assert.AreSame("request", argException.ParamName); return(true); }); }
public async Task StatusCode403OfHttpGetRequest() { if (Helper.IsNetworkAvailable()) { var http = new DefaultHttpRequester(); var request = new Request { Address = new Url("http://httpbin.org/status/403"), Method = HttpMethod.Get }; using (var response = await http.RequestAsync(request, CancellationToken.None)) { Assert.IsNotNull(response); Assert.AreEqual(403, (int)response.StatusCode); Assert.IsTrue(response.Content.CanRead); Assert.IsTrue(response.Headers.Count > 0); } } }
public async Task MethodNotAllowedOnHttpPut() { if (Helper.IsNetworkAvailable()) { var http = new DefaultHttpRequester(); var request = new Request { Address = new Url("http://httpbin.org/get"), Method = HttpMethod.Put, Content = Helper.StreamFromString("Should be ignored") }; using (var response = await http.RequestAsync(request, CancellationToken.None)) { Assert.IsNotNull(response); Assert.AreEqual(405, (int)response.StatusCode); Assert.IsTrue(response.Content.CanRead); Assert.IsTrue(response.Headers.Count > 0); } } }
public void ParseSite() { try { dbWorker.ResetActualState(); //Сбрасываем значения актуальности var requester = new DefaultHttpRequester(); requester.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36"; var config = Configuration.Default.With(requester).WithDefaultLoader(); //Открываем первую страницу по ссылке с параметрами поиска. На Авито суммарный результат найденного хранится в span[data-marker='page-title/count'] var document = BrowsingContext.New(config).OpenAsync(cModel.link); var result_count = (document.Result.QuerySelector("span[data-marker='page-title/count']") != null) ? Convert.ToInt32(document.Result.QuerySelector("span[data-marker='page-title/count']").TextContent) : 0; int parse_counter = 0; //Счетчик найденных элементов int page_counter = 1; //Счетчик страниц while (parse_counter < result_count) { Console.WriteLine("Page: " + page_counter + "..."); //Переход к конкретной странице var page_documet = BrowsingContext.New(config).OpenAsync(cModel.link + "&p=" + page_counter); //Проверяем если количество изменилось result_count = page_documet.Result.QuerySelector("span[data-marker='page-title/count']") == null ? 0 : Convert.ToInt32(document.Result.QuerySelector("span[data-marker='page-title/count']").TextContent); //Список нужных элементов var item_list = (page_documet.Result.QuerySelector("div[data-marker='catalog-serp']") != null) ? page_documet.Result.QuerySelector("div[data-marker='catalog-serp']").Children .Where(s => s.GetAttribute("data-marker") == "item").ToList() : null; if (item_list == null) { Console.WriteLine("Can't parse page: " + page_counter); throw new Exception("Can't parse page"); } foreach (var item in item_list) { //Ищем id и ссылку каждого элемента int? item_id = (item.GetAttribute("data-item-id") != null) ? Convert.ToInt32(item.GetAttribute("data-item-id")) : null; string link = item.QuerySelector("a[itemprop='url']").GetAttribute("href") ?? null; if (item_id == null || link == null) { Console.WriteLine("Can't parse item on: " + page_counter + " " + item.TagName); continue; } var find_item = dbWorker.GetByItemID((int)item_id); if (find_item == null) { AvitoDBModel newItem = new AvitoDBModel(); newItem.link = link; newItem.product_id = (int)item_id; newItem.actual = true; int res = dbWorker.WriteItem(newItem); /* cBot.setChatId(cModel.chatId); * cBot.setMessage("https://www.avito.ru" + link);*/ Console.WriteLine("Add new item " + link); } else { //Если такая позиция существует в базе -> подтверждаем актуальность find_item.actual = true; dbWorker.UpdateItem(find_item); } parse_counter++; } page_counter++; } dbWorker.ClearNonActual(); // Удаляем всё, что не актуально Console.WriteLine("Success!"); } catch { throw new Exception("Cant parse site"); } }