public HttpMockState()
{
// dependencies
TestHandler = new HttpMockHandler(this);
HttpClient = new HttpClient(TestHandler);
// data
Request = new Request
{
Method = AngleSharp.Network.HttpMethod.Post,
Address = new Url("http://example/path?query=value"),
Headers = new Dictionary<String, String>
{
{"User-Agent", "Foo/2.0"},
{"Cookie", "foo=bar"},
{"Content-Type", "application/json"},
{"Content-Length", "9"}
},
Content = new MemoryStream(Encoding.UTF8.GetBytes("\"request\""))
};
HttpResponseMessage = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("\"response\"", Encoding.UTF8, "application/json"),
Headers =
{
{"Server", "Fake"},
{"X-Powered-By", "Magic"},
{"X-CSV", new[] {"foo", "bar"}}
}
};
// setup
Target = new HttpClientRequester(HttpClient);
}
/// <summary>
/// Setup the Uri requester complete with user agent etc.
/// </summary>
/// <returns>Task that completes when the requester is ready to use.</returns>
protected override Task SetupRequesterAsync()
{
var httpClientRequester = new HttpClientRequester();
httpClientRequester.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(ClientUserAgent);
var systemUserAgent = WindowsSystemInfo.GetSystemUserAgent();
if (!String.IsNullOrEmpty(systemUserAgent))
{
httpClientRequester.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(systemUserAgent);
}
Requester = httpClientRequester.Request;
return(Task.FromResult(true));
}
/// <summary>
/// Setup the Uri requester complete with user agent etc.
/// </summary>
/// <returns>Task that completes when the requester is ready to use.</returns>
protected override void SetupRequester()
{
var httpClientRequester = new HttpClientRequester();
httpClientRequester.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(ClientUserAgent);
var systemUserAgent = WindowsSystemInfo.GetSystemUserAgent();
if (!String.IsNullOrEmpty(systemUserAgent))
{
httpClientRequester.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(systemUserAgent);
}
Requester = httpClientRequester.Request;
//Note(LC): This doesn't seem needed anymore
//return Task.FromResult(true);
}
public static async Task MainAsync(string[] args)
{
// setup
var httpClient = new HttpClient();
var requester = new HttpClientRequester(httpClient);
var configuration = new Configuration(new[] { new LoaderService(new[] { requester }) });
var context = BrowsingContext.New(configuration);
// request
var request = DocumentRequest.Get(Url.Create("http://httpbin.org/html"));
var response = await context.Loader.LoadAsync(request, CancellationToken.None);
// parse
var document = await context.OpenAsync(response, CancellationToken.None);
// interact
Console.WriteLine(document.QuerySelector("h1").ToHtml());
}
public async Task EndToEnd()
{
if (Helper.IsNetworkAvailable())
{
// ARRANGE
var httpClient = new HttpClient();
var requester = new HttpClientRequester(httpClient);
var configuration = Configuration.Default.With(requester).WithDefaultLoader();
var context = BrowsingContext.New(configuration);
var request = DocumentRequest.Get(Url.Create("http://httpbin.org/html"));
// ACT
var response = await context.GetService <IDocumentLoader>().FetchAsync(request).Task;
var document = await context.OpenAsync(response, CancellationToken.None);
// ASSERT
document.QuerySelector("h1").ToHtml().Should().Be("<h1>Herman Melville - Moby-Dick</h1>");
}
}
public static async Task <CookieCollection> GetCollectitonAsync()
{
var collection = new CookieCollection();
var cookiesFile = "data/cookies.txt";
if (File.Exists(cookiesFile))
{
var readAllText = File.ReadAllLines(cookiesFile).Where(s => !s.TrimStart().StartsWith("#"));
foreach (var line in readAllText)
{
var match = RegexCookiesTxt.Match(line);
if (match.Success)
{
try
{
var name = match.Groups[6].Value;
var value = match.Groups[7].Value;
if (name == "myCookie")
{
value = "username=&userPsw=";
}
collection.Add(new Cookie(name, value, match.Groups[3].Value, match.Groups[1].Value));
}
catch (Exception e)
{
Log.Warning(e.Message);
}
}
}
}
await SemaphoreSlim.WaitAsync();
if (AntiScrappingCookie == null)
{
try
{
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue {
NoCache = true
};
httpClient.DefaultRequestHeaders.TryAddWithoutValidation(
"user-agent",
ScrappingConfiguration.Agent);
var requester = new HttpClientRequester(httpClient);
var config = Configuration.Default.WithRequester(requester)
.WithDefaultLoader(new LoaderOptions {
IsResourceLoadingEnabled = true
}).WithJs();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync("https://www.tuenvio.cu/stores.json").WaitUntilAvailable();
var content = document.Body.TextContent;
if (!string.IsNullOrWhiteSpace(content))
{
var parametersMatch = RegexCall.Match(content);
if (parametersMatch.Success)
{
var cookieName = parametersMatch.Groups[1].Value.Trim();
var toNumbersACall = RegexA.Match(content).Groups[1].Value;
var toNumbersBCall = RegexB.Match(content).Groups[1].Value;
var toNumbersCCall = RegexC.Match(content).Groups[1].Value;
var parameters = parametersMatch.Groups[2].Value;
parameters = parameters.Replace("a", "%A%").Replace("b", "%B%").Replace("c", "%C%");
parameters = parameters.Replace("%A%", toNumbersACall).Replace("%B%", toNumbersBCall)
.Replace("%C%", toNumbersCCall);
// Review: looks like the WaitUntilAvailable method is not working properly.
var cookieValue = string.Empty;
while (string.IsNullOrWhiteSpace(cookieValue))
{
try
{
cookieValue = document.ExecuteScript($"toHex(slowAES.decrypt({parameters}))")
.ToString();
}
catch (Exception e)
{
Log.Warning(e, "Error retrieving the global cookie");
await Task.Delay(100);
}
}
AntiScrappingCookie = new Cookie(cookieName, cookieValue, "/", "www.tuenvio.cu");
}
}
}
catch (Exception e)
{
Log.Warning(e, "Error evaluating the anti-scrapping cookie.");
}
}
SemaphoreSlim.Release();
var cookie = collection.FirstOrDefault(c => c.Name == AntiScrappingCookie.Name);
if (cookie != null)
{
collection.Remove(cookie);
}
collection.Add(AntiScrappingCookie);
return(collection);
}
static async Task Test()
{
try
{
WebProxy WP = new WebProxy("61.183.233.6", 54896);
ICredentials jxCredt = new NetworkCredential("proxy_username", "proxy_password");
WP.Credentials = jxCredt;
string requestUrl = "http://csinvoice.bsgoal.net.cn/home/about";
var u = new Uri(requestUrl);
var webreq = HttpWebRequest.Create(u);
webreq.Proxy = WP;//将代理赋值给HttpWebRequest的Proxy属性
var p = await webreq.GetResponseAsync();
//HttpHost proxy = new HttpHost("你的代理的IP", 8080, "http");
var html = p.ResponseUri.ToString();
using (var reader = new StreamReader(p.GetResponseStream()))
{
string data = await reader.ReadToEndAsync();
}
string proxyUri = string.Format("{0}:{1}", "58.53.128.83", "3128");
NetworkCredential proxyCreds = new NetworkCredential(
"xxxx",
"xxxx"
);
WebProxy proxy = new WebProxy(proxyUri, false)
{
UseDefaultCredentials = false,
Credentials = proxyCreds,
};
var cookieContainer = new CookieContainer();
var handler = new HttpClientHandler()
{
Proxy = proxy,
PreAuthenticate = true,
UseDefaultCredentials = false,
CookieContainer = cookieContainer
};
var httpClient = new HttpClient(handler);
//var result =await client.GetAsync(requestUrl);
string e = string.Empty;
using (HttpResponseMessage response = await httpClient.GetAsync(requestUrl))
{
e = await response.Content.ReadAsStringAsync();
}
//httpClient.h
//httpClient.DefaultRequestHeaders.Add("X-Forwarded-For", "162.150.10.16");
//httpClient.
var Requester = new HttpClientRequester(httpClient);
var Configuration = AngleSharp.Configuration.Default.WithDefaultLoader(setup =>
{
setup.IsResourceLoadingEnabled = true;
}, requesters: new[] { Requester }).WithJavaScript().WithCss();
var config = Configuration.Default.WithDefaultLoader();
}
catch (Exception ex)
{
}
}
public async Task EndToEnd()
{
// ARRANGE
var httpClient = new HttpClient();
var requester = new HttpClientRequester(httpClient);
var configuration = new Configuration(new[] { new LoaderService(new[] { requester }) });
var context = BrowsingContext.New(configuration);
var request = DocumentRequest.Get(Url.Create("http://httpbin.org/html"));
// ACT
var response = await context.Loader.LoadAsync(request, CancellationToken.None);
var document = await context.OpenAsync(response, CancellationToken.None);
// ASSERT
document.QuerySelector("h1").ToHtml().Should().Be("<h1>Herman Melville - Moby-Dick</h1>");
}
public async Task EndToEnd()
{
if (Helper.IsNetworkAvailable())
{
// ARRANGE
var httpClient = new HttpClient();
var requester = new HttpClientRequester(httpClient);
var configuration = Configuration.Default.WithDefaultLoader(requesters: new[] { requester });
var context = BrowsingContext.New(configuration);
var request = DocumentRequest.Get(Url.Create("http://httpbin.org/html"));
// ACT
var response = await context.Loader.DownloadAsync(request).Task;
var document = await context.OpenAsync(response, CancellationToken.None);
// ASSERT
document.QuerySelector("h1").ToHtml().Should().Be("<h1>Herman Melville - Moby-Dick</h1>");
}
}
/// <summary>
/// Create a new <see cref="FaucetPayClient"/> with the default requester.
/// </summary>
/// <remarks>
/// The default <see cref="HttpClient"/> should be disposed using <see cref="Dispose"/>,
/// but a custom <see cref="HttpClient"/> is not disposed, even within <see cref="Dispose"/>.
/// </remarks>
/// <param name="config">The configuration used for the client</param>
/// <param name="client">
/// A <see cref="HttpClient"/> used for requests, when set to null, a new instance will be used.
/// </param>
/// <returns>A <see cref="FaucetPayClient"/> instance.</returns>
public static FaucetPayClient Create(ApiConfig config, HttpClient client = null)
{
var requester = HttpClientRequester.Create(config, client);
return(new FaucetPayClient(requester));
}
private async Task <Cookie> ReadAntiScrappingCookieAsync()
{
Cookie antiScrappingCookie = null;
Log.Information("Initializing Anti-Scrapping Cookie.");
try
{
var httpClient = new HttpClient {
Timeout = TimeSpan.FromSeconds(60)
};
httpClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue {
NoCache = true
};
httpClient.DefaultRequestHeaders.TryAddWithoutValidation(
"user-agent",
ScraperConfigurations.GetSupportedAgent());
var requester = new HttpClientRequester(httpClient);
var config = Configuration.Default.WithRequester(requester)
.WithDefaultLoader(new LoaderOptions {
IsResourceLoadingEnabled = true
}).WithJs();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(ScraperConfigurations.StoresJsonUrl).WaitUntilAvailable();
var content = document.Body.TextContent;
var match = Regex.Match(content, @"Server\sError\s+406");
if (!match.Success && !string.IsNullOrWhiteSpace(content))
{
var parametersMatch = this.RegexCall.Match(content);
if (parametersMatch.Success)
{
var cookieName = parametersMatch.Groups[1].Value.Trim();
var toNumbersACall = this.RegexA.Match(content).Groups[1].Value;
var toNumbersBCall = this.RegexB.Match(content).Groups[1].Value;
var toNumbersCCall = this.RegexC.Match(content).Groups[1].Value;
var parameters = parametersMatch.Groups[2].Value;
parameters = parameters.Replace("a", "%A%").Replace("b", "%B%").Replace("c", "%C%");
parameters = parameters.Replace("%A%", toNumbersACall).Replace("%B%", toNumbersBCall)
.Replace("%C%", toNumbersCCall);
// Review: looks like the WaitUntilAvailable method is not working properly.
var cookieValue = string.Empty;
while (string.IsNullOrWhiteSpace(cookieValue))
{
try
{
cookieValue = document.ExecuteScript($"toHex(slowAES.decrypt({parameters}))")
.ToString();
}
catch (Exception e)
{
Log.Warning(e, "Error retrieving the Anti-Scrapping cookie.");
await Task.Delay(100);
}
}
Log.Information(
"Read cookie '{CookieName}' with value '{CookieValue}'.",
cookieName,
cookieValue);
antiScrappingCookie = new Cookie(cookieName, cookieValue, "/", "www.tuenvio.cu");
}
}
}
catch (Exception e)
{
Log.Warning(e, "Error evaluating the Anti-Scrapping cookie.");
}
return(antiScrappingCookie);
}
