public async Task AsyncHttpGetRequest()
{
if (Helper.IsNetworkAvailable())
{
var http = new DefaultHttpRequester();
var request = new Request
{
Address = new Url("http://httpbin.org/robots.txt"),
Method = HttpMethod.Get
};
using (var response = http.RequestAsync(request, CancellationToken.None))
{
Assert.IsNotNull(response);
Assert.IsFalse(response.IsCompleted);
var result = await response;
Assert.IsTrue(response.IsCompleted);
Assert.IsTrue(result.Content.CanRead);
Assert.IsTrue(result.Headers.Count > 0);
var content = new StreamReader(result.Content);
Assert.AreEqual("User-agent: *\nDisallow: /deny\n", content.ReadToEnd());
}
}
}
public AngleSharpDriverRequester()
{
var configuration = GetBasicConfiguration();
defaultHttpRequester = new DefaultHttpRequester();
this.client = BrowsingContext.New(configuration);
}
private async Task CreateContext()
{
if (context != null)
{
return;
}
Console.WriteLine("Notabenoid login");
var requester = new DefaultHttpRequester();
requester.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0";
context = BrowsingContext.New(Configuration.Default.With(requester).WithDefaultLoader().WithDefaultCookies());
// Login
{
var queryDocument = await GetDocumentAsync("http://notabenoid.org/");
var form = queryDocument.QuerySelector <IHtmlFormElement>("form");
await form.SubmitAsync(new Dictionary <string, string> {
{ "login[login]", _notabenoidLogin },
{ "login[pass]", _notabenoidPassword },
});
}
}
public async Task RequestUserAgentString()
{
if (Helper.IsNetworkAvailable())
{
var agent = "MyAgent";
var http = new DefaultHttpRequester(agent);
var request = new Request
{
Address = new Url("http://httpbin.org/user-agent"),
Method = HttpMethod.Get
};
using (var response = await http.RequestAsync(request, CancellationToken.None))
{
Assert.IsNotNull(response);
Assert.AreEqual(200, (int)response.StatusCode);
Assert.IsTrue(response.Content.CanRead);
Assert.IsTrue(response.Headers.Count > 0);
var stream = new StreamReader(response.Content);
Assert.IsNotNull(stream);
var content = stream.ReadToEnd();
Assert.IsTrue(content.Length > 0);
Assert.AreEqual("{\n \"user-agent\": \"" + agent + "\"\n}\n", content);
}
}
}
public async Task SimpleHttpPutRequest()
{
if (Helper.IsNetworkAvailable())
{
var http = new DefaultHttpRequester();
var request = new Request
{
Address = new Url("http://httpbin.org/put"),
Method = HttpMethod.Put,
Content = Helper.StreamFromString("PUT THIS THING BACK")
};
using (var response = await http.RequestAsync(request, CancellationToken.None))
{
Assert.IsNotNull(response);
Assert.AreEqual(200, (int)response.StatusCode);
Assert.IsTrue(response.Content.CanRead);
Assert.IsTrue(response.Headers.Count > 0);
var stream = new StreamReader(response.Content);
Assert.IsNotNull(stream);
var content = stream.ReadToEnd();
Assert.IsTrue(content.Length > 0);
Assert.IsTrue(content.Contains("\"data\": \"PUT THIS THING BACK\""));
}
}
}
public void ShouldReturnHttpWebResponseAsync()
{
//Arrange
var defaultHttpRequester = new DefaultHttpRequester();
var webRequest = WebRequest.Create("https://www.google.fr");
//Act
var response = defaultHttpRequester.GetResponseAsync(webRequest);
//Assert
Assert.IsNotNull(response.Result);
}
public void TestRetrieveTokenWithWrong()
{
var requester = new DefaultHttpRequester();
var context = new ApiContext();
context.AddParam("grant_type", "client_credential")
.AddParam("appid", "wrong AppId")
.AddParam("secret", "wrong AppSecret");
Assert.Throws <ApiInvokeException>(() => {
var token = requester.Execute <Token>("/cgi-bin/token", context);
});
}
public void ShouldThrowArgumentNullException()
{
//Arrange
var defaultHttpRequester = new DefaultHttpRequester();
//Act
var ex = Assert.ThrowsException <ArgumentNullException>(() =>
{
defaultHttpRequester.GetResponse(null);
});
//Assert
Assert.AreEqual("request", ex.ParamName);
}
/// <summary>
/// Creates a requester to be used for the HTTP requests.
/// </summary>
/// <returns></returns>
private IRequester CreateRequester()
{
// Create default requester
var requester = new DefaultHttpRequester()
{
Timeout = TimeSpan.FromSeconds(10),
};
// Set headers
requester.Headers[HeaderNames.UserAgent] = Consts.USER_AGENT;
requester.Headers["dnt"] = "1";
return(requester);
}
public void TestRetrieveToken()
{
var x = new SecretInformationReader();
var requester = new DefaultHttpRequester();
var context = new ApiContext();
context.AddParam("grant_type", "client_credential")
.AddParam("appid", x.AppId)
.AddParam("secret", x.AppSecret);
var token = requester.Execute <Token>("/cgi-bin/token", context);
Assert.NotNull(token.access_token);
}
private async Task <List <dynamic> > GetPageData(string url, List <dynamic> results)
{
var requester = new DefaultHttpRequester("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:68.0) Gecko/20100101 Firefox/68.0");
var config = Configuration.Default.With(requester).WithDefaultLoader().WithDefaultCookies();
var context = BrowsingContext.New(config);
try
{
IDocument document;
if (url.Contains(AmazonUrl))
{
//var productId = url.Substring(url.LastIndexOf("/", StringComparison.Ordinal) + 1);
var productId = url[(url.LastIndexOf("/", StringComparison.Ordinal) + 1)..];
private static async Task <Dictionary <string, string> > GetHTMLGoogle(string ticker)
{
var quoteSummary = new Dictionary <string, string>();
var requester = new DefaultHttpRequester();
requester.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36";
var config = Configuration.Default.With(requester).WithDefaultLoader();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync($"https://www.google.com/search?q={ticker}");
var quoteSummaryListItems = document.QuerySelector("span[jsname='vWLAgc']");
var tmp = quoteSummaryListItems.TextContent.Replace(",", "");
quoteSummary.Add("CURRENT PRICE", tmp);
return(quoteSummary);
}
public void ShouldThrowArgumentNullExceptionAsync()
{
//Arrange
var defaultHttpRequester = new DefaultHttpRequester();
//Act
var ex = Assert.ThrowsException <AggregateException>(() =>
{
defaultHttpRequester.GetResponseAsync(null).Wait();
});
//Assert
ex.Handle((Exception arg) =>
{
var argException = arg as ArgumentNullException;
Assert.IsNotNull(argException);
Assert.AreSame("request", argException.ParamName);
return(true);
});
}
public async Task StatusCode403OfHttpGetRequest()
{
if (Helper.IsNetworkAvailable())
{
var http = new DefaultHttpRequester();
var request = new Request
{
Address = new Url("http://httpbin.org/status/403"),
Method = HttpMethod.Get
};
using (var response = await http.RequestAsync(request, CancellationToken.None))
{
Assert.IsNotNull(response);
Assert.AreEqual(403, (int)response.StatusCode);
Assert.IsTrue(response.Content.CanRead);
Assert.IsTrue(response.Headers.Count > 0);
}
}
}
public async Task MethodNotAllowedOnHttpPut()
{
if (Helper.IsNetworkAvailable())
{
var http = new DefaultHttpRequester();
var request = new Request
{
Address = new Url("http://httpbin.org/get"),
Method = HttpMethod.Put,
Content = Helper.StreamFromString("Should be ignored")
};
using (var response = await http.RequestAsync(request, CancellationToken.None))
{
Assert.IsNotNull(response);
Assert.AreEqual(405, (int)response.StatusCode);
Assert.IsTrue(response.Content.CanRead);
Assert.IsTrue(response.Headers.Count > 0);
}
}
}
public void ParseSite()
{
try
{
dbWorker.ResetActualState(); //Сбрасываем значения актуальности
var requester = new DefaultHttpRequester();
requester.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36";
var config = Configuration.Default.With(requester).WithDefaultLoader();
//Открываем первую страницу по ссылке с параметрами поиска. На Авито суммарный результат найденного хранится в span[data-marker='page-title/count']
var document = BrowsingContext.New(config).OpenAsync(cModel.link);
var result_count = (document.Result.QuerySelector("span[data-marker='page-title/count']") != null)
? Convert.ToInt32(document.Result.QuerySelector("span[data-marker='page-title/count']").TextContent) : 0;
int parse_counter = 0; //Счетчик найденных элементов
int page_counter = 1; //Счетчик страниц
while (parse_counter < result_count)
{
Console.WriteLine("Page: " + page_counter + "...");
//Переход к конкретной странице
var page_documet = BrowsingContext.New(config).OpenAsync(cModel.link + "&p=" + page_counter);
//Проверяем если количество изменилось
result_count = page_documet.Result.QuerySelector("span[data-marker='page-title/count']") == null
? 0 : Convert.ToInt32(document.Result.QuerySelector("span[data-marker='page-title/count']").TextContent);
//Список нужных элементов
var item_list = (page_documet.Result.QuerySelector("div[data-marker='catalog-serp']") != null) ? page_documet.Result.QuerySelector("div[data-marker='catalog-serp']").Children
.Where(s => s.GetAttribute("data-marker") == "item").ToList() : null;
if (item_list == null)
{
Console.WriteLine("Can't parse page: " + page_counter);
throw new Exception("Can't parse page");
}
foreach (var item in item_list)
{
//Ищем id и ссылку каждого элемента
int? item_id = (item.GetAttribute("data-item-id") != null) ? Convert.ToInt32(item.GetAttribute("data-item-id")) : null;
string link = item.QuerySelector("a[itemprop='url']").GetAttribute("href") ?? null;
if (item_id == null || link == null)
{
Console.WriteLine("Can't parse item on: " + page_counter + " " + item.TagName);
continue;
}
var find_item = dbWorker.GetByItemID((int)item_id);
if (find_item == null)
{
AvitoDBModel newItem = new AvitoDBModel();
newItem.link = link;
newItem.product_id = (int)item_id;
newItem.actual = true;
int res = dbWorker.WriteItem(newItem);
/* cBot.setChatId(cModel.chatId);
* cBot.setMessage("https://www.avito.ru" + link);*/
Console.WriteLine("Add new item " + link);
}
else
{
//Если такая позиция существует в базе -> подтверждаем актуальность
find_item.actual = true;
dbWorker.UpdateItem(find_item);
}
parse_counter++;
}
page_counter++;
}
dbWorker.ClearNonActual(); // Удаляем всё, что не актуально
Console.WriteLine("Success!");
}
catch
{
throw new Exception("Cant parse site");
}
}
