public static string PostData(string url, Dictionary <string, string> data, int contentType = 0, int userAgent = 0, Dictionary <string, string> headers = null, bool isDecode = true)
{
try
{
var cookies = new CookieContainer();
var handler = new HttpClientHandler {
CookieContainer = cookies
};
using (var myHttpWebRequest = new HttpClient(handler)
{
Timeout = new TimeSpan(0, 0, 10)
})
{
myHttpWebRequest.DefaultRequestHeaders.Add("Method", "POST");
myHttpWebRequest.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
myHttpWebRequest.DefaultRequestHeaders.Add("ContentType",
userAgent == 0 ? "application/x-www-form-urlencoded" : "application/json;charset=UTF-8");
switch (userAgent)
{
case 1:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3");
break;
case 2:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19");
break;
case 3:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)");
break;
case 4:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "NativeHost");
break;
case 5:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Apache-HttpClient/UNAVAILABLE (java 1.4)");
break;
case 6:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (iPad; CPU OS 8_1_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B466 Safari/600.1.4");
break;
default:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
break;
}
if (headers != null)
{
foreach (var k in headers)
{
if (k.Key == "Cookie")
{
cookies.SetCookies(new Uri(url), k.Value.Replace(";", ","));
}
else
{
myHttpWebRequest.DefaultRequestHeaders.Add(k.Key, k.Value);
}
}
}
var response = contentType == 0
? myHttpWebRequest.PostAsync(url, new FormUrlEncodedContent(data)).Result
: myHttpWebRequest.PostAsync(url, new StringContent(data[data.Keys.First()], Encoding.UTF8)).Result;
return(response.Content.ReadAsStringAsync().Result);
}
}
catch (Exception ex)
{
AddLog(ex.ToString());
return(null);
}
}
/// <summary>
/// Returns a SharePoint on-premises / SharePoint Online ClientContext object. Requires claims based authentication with FedAuth cookie.
/// </summary>
/// <param name="siteUrl">Site for which the ClientContext object will be instantiated</param>
/// <param name="icon">Optional icon to use for the popup form</param>
/// <returns>ClientContext to be used by CSOM code</returns>
public ClientContext GetWebLoginClientContext(string siteUrl, System.Drawing.Icon icon = null)
{
var authCookiesContainer = new CookieContainer();
var siteUri = new Uri(siteUrl);
var thread = new Thread(() =>
{
var form = new System.Windows.Forms.Form();
if (icon != null)
{
form.Icon = icon;
}
var browser = new System.Windows.Forms.WebBrowser
{
ScriptErrorsSuppressed = true,
Dock = DockStyle.Fill
};
form.SuspendLayout();
form.Width = 900;
form.Height = 500;
form.Text = $"Log in to {siteUrl}";
form.Controls.Add(browser);
form.ResumeLayout(false);
browser.Navigate(siteUri);
browser.Navigated += (sender, args) =>
{
if (siteUri.Host.Equals(args.Url.Host))
{
var cookieString = CookieReader.GetCookie(siteUrl).Replace("; ", ",").Replace(";", ",");
// Get FedAuth and rtFa cookies issued by ADFS when accessing claims aware applications.
// - or get the EdgeAccessCookie issued by the Web Application Proxy (WAP) when accessing non-claims aware applications (Kerberos).
IEnumerable <string> authCookies = null;
if (Regex.IsMatch(cookieString, "FedAuth", RegexOptions.IgnoreCase))
{
authCookies = cookieString.Split(',').Where(c => c.StartsWith("FedAuth", StringComparison.InvariantCultureIgnoreCase) || c.StartsWith("rtFa", StringComparison.InvariantCultureIgnoreCase));
}
else if (Regex.IsMatch(cookieString, "EdgeAccessCookie", RegexOptions.IgnoreCase))
{
authCookies = cookieString.Split(',').Where(c => c.StartsWith("EdgeAccessCookie", StringComparison.InvariantCultureIgnoreCase));
}
if (authCookies != null)
{
authCookiesContainer.SetCookies(siteUri, string.Join(",", authCookies));
form.Close();
}
}
};
form.Focus();
form.ShowDialog();
browser.Dispose();
});
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
thread.Join();
if (authCookiesContainer.Count > 0)
{
var ctx = new ClientContext(siteUrl);
ctx.ExecutingWebRequest += (sender, e) => e.WebRequestExecutor.WebRequest.CookieContainer = authCookiesContainer;
return(ctx);
}
return(null);
}
/// <summary>
/// Sets the cookie value for the given address.
/// </summary>
/// <param name="origin">The origin of the cookie (Url).</param>
/// <param name="value">The value of the cookie.</param>
public void SetCookie(String origin, String value)
{
var cookies = Sanatize(value);
_container.SetCookies(new Uri(origin), cookies);
}
async Task <string> DownloadPageAsync(Target source)
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(source.Url);
req.Timeout = Constants.WebRequestMillisecondTimeout;
req.Method = source.WebRequestMethod;
req.AuthenticationLevel = System.Net.Security.AuthenticationLevel.None; //TODO authentication to be implemented
InternalLog(LogType.Info, $"Downloading {source.Url}");
if (source.UseHeaders)
{
InternalLog(LogType.Info, $"Using Headers for {source.Url}");
if (!String.IsNullOrWhiteSpace(source.Cookie))
{
CookieContainer cookieContainer = new CookieContainer();
cookieContainer.SetCookies(new Uri(source.Url), source.Cookie);
}
if (!string.IsNullOrWhiteSpace(source.UserAgent))
{
req.Headers.Add(HttpRequestHeader.UserAgent, source.UserAgent);
}
if (!string.IsNullOrWhiteSpace(source.Accept))
{
req.Headers.Add(HttpRequestHeader.Accept, source.Accept);
}
if (!string.IsNullOrWhiteSpace(source.AcceptLanguage))
{
req.Headers.Add(HttpRequestHeader.AcceptLanguage, source.AcceptLanguage);
}
if (!string.IsNullOrWhiteSpace(source.CacheControl))
{
req.Headers.Add(HttpRequestHeader.CacheControl, source.CacheControl);
}
if (!string.IsNullOrWhiteSpace(source.Host))
{
req.Headers.Add(HttpRequestHeader.Host, source.Host);
}
if (!string.IsNullOrWhiteSpace(source.AcceptEncoding))
{
req.Headers.Add(HttpRequestHeader.AcceptEncoding, source.AcceptEncoding);
}
}
if (source.UseCompression)
{
InternalLog(LogType.Info, $"Using Compression for {source.Url}");
req.Headers[HttpRequestHeader.AcceptEncoding] = Constants.WebRequestCompressionEncoding;
req.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
}
try
{
using (var wresponse = await req.GetResponseAsync().ConfigureAwait(true))
{
using (StreamReader reader = new StreamReader(wresponse.GetResponseStream(), Encoding.Default))
{
return(reader.ReadToEnd());
}
}
}
catch (Exception e)
{
InternalLog(LogType.Error, $"Error {e.Message}");
InternalLog(LogType.Error, e.StackTrace);
return(String.Empty);
}
}
/// <summary>
/// Sets the cookie value for the given address.
/// </summary>
/// <param name="origin">The origin of the cookie (Url).</param>
/// <param name="value">The value of the cookie.</param>
public void SetCookie(String origin, String value)
{
_container.SetCookies(new Uri(origin), value);
}
async Task <HttpResponseMessage> DoProcessRequest(HttpRequestMessage request, URL javaUrl, HttpURLConnection httpConnection, CancellationToken cancellationToken)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"{this}.DoProcessRequest ()");
}
try {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $" connecting");
}
await httpConnection.ConnectAsync().ConfigureAwait(false);
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $" connected");
}
} catch (Java.Net.ConnectException ex) {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Connection exception {ex}");
}
// Wrap it nicely in a "standard" exception so that it's compatible with HttpClientHandler
throw new WebException(ex.Message, ex, WebExceptionStatus.ConnectFailure, null);
}
if (httpConnection.DoOutput)
{
await request.Content.CopyToAsync(httpConnection.OutputStream).ConfigureAwait(false);
}
var statusCode = (HttpStatusCode)httpConnection.ResponseCode;
var connectionUri = new Uri(httpConnection.URL.ToString());
// If the request was redirected we need to put the new URL in the request
request.RequestUri = connectionUri;
var ret = new AndroidHttpResponseMessage(javaUrl, httpConnection)
{
RequestMessage = request,
ReasonPhrase = httpConnection.ResponseMessage,
StatusCode = statusCode,
};
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Status code: {statusCode}");
}
if (statusCode == HttpStatusCode.Unauthorized || statusCode == HttpStatusCode.ProxyAuthenticationRequired)
{
// We don't resend the request since that would require new set of credentials if the
// ones provided in Credentials are invalid (or null) and that, in turn, may require asking the
// user which is not something that should be taken care of by us and in this
// context. The application should be responsible for this.
// HttpClientHandler throws an exception in this instance, but I think it's not a good
// idea. We'll return the response message with all the information required by the
// application to fill in the blanks and provide the requested credentials instead.
//
// We should return the body of the response too but, alas, the Java client will throw
// a, wait for it, FileNotFound exception if we attempt to access the input stream. So
// no body, just a dummy. Java FTW!
ret.Content = new StringContent("Unauthorized", Encoding.ASCII);
CopyHeaders(httpConnection, ret);
if (ret.Headers.WwwAuthenticate != null)
{
ProxyAuthenticationRequested = false;
CollectAuthInfo(ret.Headers.WwwAuthenticate);
}
else if (ret.Headers.ProxyAuthenticate != null)
{
ProxyAuthenticationRequested = true;
CollectAuthInfo(ret.Headers.ProxyAuthenticate);
}
ret.RequestedAuthentication = RequestedAuthentication;
return(ret);
}
if (!IsErrorStatusCode(statusCode))
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Reading...");
}
Stream inputStream = new BufferedStream(httpConnection.InputStream);
if (decompress_here)
{
string[] encodings = httpConnection.ContentEncoding?.Split(',');
if (encodings != null)
{
if (encodings.Contains(GZIP_ENCODING, StringComparer.OrdinalIgnoreCase))
{
inputStream = new GZipStream(inputStream, CompressionMode.Decompress);
}
else if (encodings.Contains(DEFLATE_ENCODING, StringComparer.OrdinalIgnoreCase))
{
inputStream = new DeflateStream(inputStream, CompressionMode.Decompress);
}
}
}
ret.Content = new StreamContent(inputStream);
}
else
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Status code is {statusCode}, returning empty content");
}
// For 400 >= response code <= 599 the Java client throws the FileNotFound exeption when attempting to read from the connection
// Client tests require we return no content here
ret.Content = new StringContent(String.Empty, Encoding.ASCII);
}
CopyHeaders(httpConnection, ret);
IEnumerable <string> cookieHeaderValue;
if (!UseCookies || CookieContainer == null || !ret.Headers.TryGetValues("Set-Cookie", out cookieHeaderValue) || cookieHeaderValue == null)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"No cookies");
}
return(ret);
}
try {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Parsing cookies");
}
CookieContainer.SetCookies(connectionUri, String.Join(",", cookieHeaderValue));
} catch (Exception ex) {
// We don't want to terminate the response because of a bad cookie, hence just reporting
// the issue. We might consider adding a virtual method to let the user handle the
// issue, but not sure if it's really needed. Set-Cookie header will be part of the
// header collection so the user can always examine it if they spot an error.
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Failed to parse cookies in the server response. {ex.GetType ()}: {ex.Message}");
}
}
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Returning");
}
return(ret);
}
public string Open(string uri, string postData, string referer)
{
HttpWebRequest _request;
int i = 0;
var sw = new Stopwatch();
sw.Start();
//WaitAMoment();
//lock (_lockOpenSerialize)
{
string html = null;
HttpWebResponse response = null;
Stream responseStream = null;
StreamReader sr = null;
try
{
if (RequestInterval > 0)
{
TimeSpan ts = new TimeSpan(DateTime.Now.Ticks - _ticks);
double sleep = ts.TotalMilliseconds;
if (sleep < RequestInterval)
{
Thread.Sleep((int)(RequestInterval - sleep));
}
_ticks = DateTime.Now.Ticks;
}
if (referer == null)
{
referer = _referer;
}
if (BeginRequest != null)
{
HttpClientEventArgs e = new HttpClientEventArgs();
e.Uri = uri;
e.PostData = postData;
e.Referer = referer;
BeginRequest(e);
}
if (uri.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
_request = (HttpWebRequest)WebRequest.Create(uri);
_request.ProtocolVersion = HttpVersion.Version10;
}
else
{
_request = (HttpWebRequest)WebRequest.Create(uri);
}
_request.Referer = referer;
_request.Headers.Add("x-requested-with", "XMLHttpRequest");
_request.Accept = _accept;
_request.UserAgent = _userAgent;
_request.CookieContainer = _cookieContainer;
_request.Timeout = Timeout;
if (_proxy != null)
{
_request.Proxy = _proxy;
}
_request.AllowWriteStreamBuffering = true;
_request.AllowAutoRedirect = AllowAutoRedirect;
_request.KeepAlive = true;
if (AddRequestHeader != null)
{
foreach (var addheader in AddRequestHeader)
{
_request.Headers.Add(addheader.Key, addheader.Value);
}
}
foreach (Cookie cookie in _cookieCollection)
{
_cookieContainer.SetCookies(_request.RequestUri, cookie.Name + "=" + cookie.Value);
}
if (string.IsNullOrEmpty(postData))
{
_request.Method = "GET";
}
else
{
byte[] bytes = Encoding.GetEncoding(_charset).GetBytes(postData);
_request.Method = "POST";
_request.ContentType = _contentType;
_request.ContentLength = bytes.Length;
using (Stream writer = _request.GetRequestStream())
{
writer.Write(bytes, 0, bytes.Length);
}
}
response = (HttpWebResponse)_request.GetResponse();
ResponeHeader = response.Headers;
#region Cookie管理
// 1.当前地址的cookie （基础类自动处理，不会处理非当前路径的cookie）
_cookieContainer = _request.CookieContainer;
// 2. 处理response返回的所有cookie(非当前url路径的)
string allCookies = response.Headers["Set-Cookie"];
if (!string.IsNullOrWhiteSpace(allCookies))
{
string pathSplit = "path=";
while (allCookies.Contains(pathSplit))
{
#region 分解["Set-Cookie"]字符串
string cookieText =
allCookies.Substring(0, allCookies.IndexOf(pathSplit,
StringComparison.CurrentCultureIgnoreCase));
allCookies = allCookies.Substring($"{cookieText}{pathSplit}".Length);
int nextCookieSplitIndex = allCookies.IndexOf(",",
StringComparison.CurrentCultureIgnoreCase);
string cookiePath = nextCookieSplitIndex == -1
? allCookies
: allCookies.Substring(0, nextCookieSplitIndex);
allCookies = nextCookieSplitIndex == -1
? string.Empty
: allCookies.Substring($"{cookiePath},".Length);
#endregion
#region 注入 _cookieContainer
string cookiePathFull;
cookieText = cookieText.Trim();
cookiePath = cookiePath.Trim();
if (cookiePath.Contains("://"))
{
cookiePathFull = cookiePath;
}
else
{
cookiePathFull = $"{_request.RequestUri.Scheme}://{_request.RequestUri.Authority}{cookiePath}";
}
_cookieContainer.SetCookies(new Uri(cookiePathFull), cookieText);
#endregion
}
}
#endregion
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
}
else
{
responseStream = response.GetResponseStream();
}
sr = new StreamReader(responseStream, Encoding.GetEncoding(_charset));
html = sr.ReadToEnd();
if (EndRequest != null)
{
HttpClientEventArgs e = new HttpClientEventArgs();
e.Uri = uri;
e.PostData = postData;
e.Referer = referer;
e.ResponseText = html;
EndRequest(e);
}
}
catch (Exception ex)
{
//LogCache.Instance.AddInfo(
// $"HttpClient.Open() 发生异常。(url:{uri},postdata:{postData},referer{referer}) \n异常详情：{ex}\n{ex.StackTrace}");
//if (Error != null)
//{
// Error(ex);
//}
log.Error("请求发生异常", ex);
}
finally
{
_referer = uri;
if (response != null)
{
response.Close();
response = null;
}
if (responseStream != null)
{
responseStream.Close();
responseStream = null;
}
if (sr != null)
{
sr.Close();
sr = null;
}
if (html == null)
{
log.Error($"请求html为null,url:{uri},postdata:{postData}");
//LogCache.Instance.AddInfo(
// $"HttpClient.Open() 请求html为null。(invokmethod{new StackFrame(1).GetMethod()}, url:{uri},postdata:{postData},referer{referer}) ");
}
}
sw.Stop();
//log.Debug($"{i}请求服务花费了{sw.ElapsedMilliseconds}ms。");
i++;
return(html);
}
}
public void AddCookie(string key, string value)
{
_cookieContainer.SetCookies(new Uri(_baseAddress), key + "=" + value);
}
public static void SetCookies_InvalidData_Throws(Uri uri, string cookieHeader)
{
CookieContainer cc = new CookieContainer();
Assert.Throws <CookieException>(() => cc.SetCookies(uri, cookieHeader));
}
private async Task RegisterModulesOnServer(IEnumerable <ModuleInfo> modules)
{
Uri baseUri = new Uri("http://localhost:5000");
CookieContainer cookieContainer = new CookieContainer();
HttpClientHandler handler = new HttpClientHandler();
handler.CookieContainer = cookieContainer;
HttpClient getClient = new HttpClient(handler);
getClient.BaseAddress = baseUri;
getClient.DefaultRequestHeaders.Connection.Add("GET");
CookieContainer postCookieContainer = new CookieContainer();
HttpClientHandler postHandler = new HttpClientHandler();
postHandler.CookieContainer = postCookieContainer;
HttpClient postClient = new HttpClient(postHandler);
postClient.BaseAddress = baseUri;
postClient.DefaultRequestHeaders.Connection.Add("POST");
HtmlParser parser = new HtmlParser();
for (int i = 0; i < modules.Count(); i++)
{
ModuleInfo m = modules.ElementAt(i);
if (m.Name == "HelpCommand")
{
continue;
}
Module module = new Module
{
Name = m.Name,
Description = m.Summary,
Enabled = true,
Options = ""
};
string json = JsonConvert.SerializeObject(module);
HttpResponseMessage getMsg = await getClient.GetAsync("Modules/Create");
string html = await(getMsg.Content.ReadAsStringAsync());
IHtmlDocument doc = await parser.ParseAsync(html);
IElement tokenInput = doc.QuerySelector("input[name=__RequestVerificationToken]");
string token = tokenInput.Attributes ["value"].Value;
IEnumerable <Cookie> responseCookies = cookieContainer.GetCookies(new Uri("http://localhost:5000/Modules/Create")).Cast <Cookie> ();
string cookie = responseCookies.FirstOrDefault(c => c.Name == ".AspNetCore.Antiforgery.YbN9yTuP6nI").Value;
postClient.DefaultRequestHeaders.Clear();
postClient.DefaultRequestHeaders.Add("RequestVerificationToken", token);
postCookieContainer.SetCookies(new Uri(baseUri, "Modules/Create"), $".AspNetCore.Antiforgery.YbN9yTuP6nI={cookie}");
HttpResponseMessage postResponse = await postClient.PostAsync(new Uri (baseUri, "Modules/Create"), new StringContent (json, Encoding.UTF8, "application/json"));
}
}
public void Send()
{
if (sent)
{
throw new InvalidOperationException("Request has already completed.");
}
sent = true;
isDone = false;
state = RequestState.Waiting;
#if USE_GZIP
if (acceptGzip)
{
SetHeader("Accept-Encoding", "gzip");
}
#endif
ThreadPool.QueueUserWorkItem(new WaitCallback(delegate(object t) {
try {
var retry = 0;
while (++retry < maximumRedirects)
{
if (useCache)
{
string etag = "";
if (etags.TryGetValue(uri.AbsoluteUri, out etag))
{
SetHeader("If-None-Match", etag);
}
}
var hostHeader = uri.Host;
if (uri.Port != 80 && uri.Port != 443)
{
hostHeader += ":" + uri.Port.ToString();
}
SetHeader("Host", hostHeader);
if (enableCookies && uri != null)
{
try {
var c = cookies.GetCookieHeader(uri);
if (c != null && c.Length > 0)
{
SetHeader("Cookie", c);
}
} catch (NullReferenceException) {
//Some cookies make the .NET cookie class barf. MEGH again.
//Debug.Log (".NET cannot parse this cookie: " + e.ToString ());
} catch (IndexOutOfRangeException) {
//Another weird exception that comes through from the cookie class.
}
}
ActiveConnection connection;
try {
//pull a connection from the pool (a new one is created if needed)
connection = GetClient(uri.Host, uri.Port, uri.Scheme.ToLower() == "https");
} catch (Exception e) {
Debug.Log(e);
exception = e;
response = null;
break;
}
try {
WriteToStream(connection.stream);
} catch (IOException e) {
Debug.Log(e);
exception = new IOException("Server closed the connection:" + e.ToString());
response = null;
break;
}
response = new Response(this);
state = RequestState.Reading;
try {
response.ReadFromStream(connection.stream);
} catch (IOException e) {
Debug.Log(e);
exception = new IOException("Server closed the connection:" + e.ToString());
response = null;
break;
}
if (response != null)
{
if (enableCookies)
{
foreach (var i in response.GetHeaders("Set-Cookie"))
{
try {
cookies.SetCookies(uri, i);
} catch (System.Net.CookieException) {
//Some cookies make the .NET cookie class barf. MEGH.
}
}
}
switch (response.status)
{
case 101:
upgradedConnection = connection;
retry = maximumRedirects;
break;
case 304:
retry = maximumRedirects;
break;
case 307:
case 302:
case 301:
uri = new Uri(response.GetHeader("Location"));
if (OnRedirect != null)
{
OnRedirect(uri);
retry = maximumRedirects;
}
break;
default:
retry = maximumRedirects;
break;
}
//close the connection back if not upgraded.
if (upgradedConnection == null)
{
lock (connectionPool) {
var close = response.GetHeader("Connection").ToLower() == "close";
if (!close)
{
connectionPool.Add(connection);
}
else
{
connection.stream.Close();
}
}
}
}
}
if (useCache && response != null)
{
string etag = response.GetHeader("etag");
if (etag.Length > 0)
{
etags [uri.AbsoluteUri] = etag;
SaveEtags();
}
}
} catch (Exception e) {
Debug.Log(e);
exception = e;
response = null;
}
state = RequestState.Done;
isDone = true;
}));
}
/// <summary>
/// Main login function.
/// * This can be pretty much ignored, no changes are needed here.
/// </summary>
public bool Login(string u, string p)
{
cookieContainer.SetCookies(new Uri("http://age-of-aincrd.com"), "cookie_check=1;");
string[] loginParams = new string[] { u, p };
string rLoginData = string.Format("login={0}®ister=0&password={1}", loginParams);
byte[] bLoginData = Encoding.UTF8.GetBytes(rLoginData);
HttpWebRequest loginRequest = (HttpWebRequest)WebRequest.Create("https://age-of-aincrad.com/forum/login/login");
loginRequest.CookieContainer = cookieContainer;
loginRequest.Method = "POST";
loginRequest.AllowAutoRedirect = false;
loginRequest.ContentType = "application/x-www-form-urlencoded";
loginRequest.ContentLength = (long)bLoginData.Length;
loginRequest.Referer = "http://age-of-aincrad.com";
loginRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36";
Stream loginRequestStream = loginRequest.GetRequestStream();
loginRequestStream.Write(bLoginData, 0, bLoginData.Length);
loginRequestStream.Close();
HttpWebResponse loginResponse = (HttpWebResponse)loginRequest.GetResponse();
loginRequestStream = loginResponse.GetResponseStream();
StreamReader loginRequestReader = new StreamReader(loginRequestStream);
string loginRequestData = loginRequestReader.ReadToEnd();
cookieContainer.Add(loginResponse.Cookies);
if (loginResponse.StatusCode == HttpStatusCode.SeeOther)
{
HttpWebRequest tokenRequest = (HttpWebRequest)WebRequest.Create("http://age-of-aincrad.com/forum/");
tokenRequest.CookieContainer = new CookieContainer();
tokenRequest.CookieContainer.Add(loginResponse.Cookies);
loginRequestStream = tokenRequest.GetResponse().GetResponseStream();
loginRequestReader = new StreamReader(loginRequestStream);
loginRequestData = loginRequestReader.ReadToEnd();
if (loginRequestData.Contains("_xfToken"))
{
string tokenValue = loginRequestData.Substring(loginRequestData.IndexOf("_xfToken") + 17);
tokenValue = tokenValue.Split(new char[] { '"' })[0];
forumData.SetToken(tokenValue);
}
}
loginRequestReader.Close();
loginRequestStream.Close();
loginResponse.Close();
if (forumData.GetToken() != "")
{
userData.SetLoginState(true);
userData.SetUsername(u);
userData.SetPassword(p);
return(true);
}
return(false);
}
public static void setCoockies(Uri uri, string cookieHeader)
{
coockieContainer.SetCookies(uri, cookieHeader);
}
override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var useProxy = false;
WebProxy proxyServer = null;
if (Startup.ProxyConnection != null)
{
proxyServer = new WebProxy(Startup.ProxyConnection, false);
useProxy = true;
}
ClearanceHandler clearanceHandlr = new ClearanceHandler();
HttpClientHandler clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = proxyServer,
UseProxy = useProxy,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
clearanceHandlr.InnerHandler = clientHandlr;
var client = new HttpClient(clearanceHandlr);
if (webRequest.EmulateBrowser)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
foreach (var header in response.Headers)
{
IEnumerable <string> value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';') + 1))));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
public static void SetCookies_InvalidData_Throws(Uri uri, string cookieHeader)
{
CookieContainer cc = new CookieContainer();
Assert.Throws<CookieException>(() => cc.SetCookies(uri, cookieHeader));
}
public static HttpResponseMessage CreateResponseMessage(
WinHttpRequestState state,
bool doManualDecompressionCheck)
{
HttpRequestMessage request = state.RequestMessage;
SafeWinHttpHandle requestHandle = state.RequestHandle;
CookieUsePolicy cookieUsePolicy = state.Handler.CookieUsePolicy;
CookieContainer cookieContainer = state.Handler.CookieContainer;
var response = new HttpResponseMessage();
bool stripEncodingHeaders = false;
// Get HTTP version, status code, reason phrase from the response headers.
string version = GetResponseHeaderStringInfo(requestHandle, Interop.WinHttp.WINHTTP_QUERY_VERSION);
if (string.Compare("HTTP/1.1", version, StringComparison.OrdinalIgnoreCase) == 0)
{
response.Version = HttpVersion.Version11;
}
else if (string.Compare("HTTP/1.0", version, StringComparison.OrdinalIgnoreCase) == 0)
{
response.Version = HttpVersion.Version10;
}
else
{
response.Version = HttpVersion.Unknown;
}
response.StatusCode = (HttpStatusCode)GetResponseHeaderNumberInfo(
requestHandle,
Interop.WinHttp.WINHTTP_QUERY_STATUS_CODE);
response.ReasonPhrase = GetResponseHeaderStringInfo(
requestHandle,
Interop.WinHttp.WINHTTP_QUERY_STATUS_TEXT);
// Create response stream and wrap it in a StreamContent object.
var responseStream = new WinHttpResponseStream(state);
Stream decompressedStream = responseStream;
if (doManualDecompressionCheck)
{
string contentEncoding = GetResponseHeaderStringInfo(
requestHandle,
Interop.WinHttp.WINHTTP_QUERY_CONTENT_ENCODING);
if (!string.IsNullOrEmpty(contentEncoding))
{
if (contentEncoding.IndexOf(EncodingNameDeflate, StringComparison.OrdinalIgnoreCase) > -1)
{
decompressedStream = new DeflateStream(responseStream, CompressionMode.Decompress);
stripEncodingHeaders = true;
}
else if (contentEncoding.IndexOf(EncodingNameGzip, StringComparison.OrdinalIgnoreCase) > -1)
{
decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress);
stripEncodingHeaders = true;
}
}
}
var content = new StreamContent(decompressedStream);
response.Content = content;
response.RequestMessage = request;
// Parse raw response headers and place them into response message.
ParseResponseHeaders(requestHandle, response, stripEncodingHeaders);
// Store response header cookies into custom CookieContainer.
if (cookieUsePolicy == CookieUsePolicy.UseSpecifiedCookieContainer)
{
Debug.Assert(cookieContainer != null);
if (response.Headers.Contains(HeaderNameSetCookie))
{
IEnumerable <string> cookieHeaders = response.Headers.GetValues(HeaderNameSetCookie);
foreach (var cookieHeader in cookieHeaders)
{
try
{
cookieContainer.SetCookies(request.RequestUri, cookieHeader);
}
catch (CookieException)
{
// We ignore malformed cookies in the response.
}
}
}
}
return(response);
}
public static void SetCookies_Invalid()
{
CookieContainer cc = new CookieContainer();
Assert.Throws<ArgumentNullException>(() => cc.SetCookies(null, "")); //Null uri
Assert.Throws<ArgumentNullException>(() => cc.SetCookies(new Uri("http://contoso.com"), null)); //Null header
}
internal static void AddCookie(this CookieContainer cookieContainer, Uri uri, string cookieHeader) => cookieContainer.SetCookies(uri, cookieHeader);
/// <summary>
/// Universal method to load page
/// </summary>
bool _do(HttpRequest http_request, bool send_cookies)
{
Stream res_stream = null;
try
{
init_loading(http_request.Url);
if (UseCache)
{
if (Cache.GetCachedFile(http_request.Url, http_request.PostString, out binary_result, out ResponseUrl, out CachedFile))
{
web_routine_status = WebRoutineStatus.CACHED;
Log.Write("From cache: " + http_request.Url);
return(true);
}
}
init_loading_progress(http_request.Url);
if (http_request.Method == HttpRequest.RequestMethod.POST)
{
System.Net.ServicePointManager.Expect100Continue = false;
}
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(http_request.Url);
req.Credentials = credential_cache;
req.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
if (send_cookies)
{
if (UseIeCookies)
{
Uri uri = new Uri(http_request.Url);
string cookie = IeRoutines.RetrieveIeCookies(uri);
if (cookie.Length > 0)
{
cookie = Regex.Replace(cookie, ";", ",");
lock (CookieContainer)
CookieContainer.SetCookies(uri, cookie);
}
}
req.CookieContainer = CookieContainer;
}
if (Proxy != null)
{
req.Proxy = Proxy.WebProxy;
}
req.Timeout = Properties.Web.Default.HttpRequestTimeoutInSeconds * 1000;
req.ReadWriteTimeout = Properties.Web.Default.HttpRequestTimeoutInSeconds * 1000;
if (HttpRequest.MaxAutoRedirectionCount == 0)
{
req.AllowAutoRedirect = false;
}
else if (HttpRequest.MaxAutoRedirectionCount > 0)
{
req.AllowAutoRedirect = true;
req.MaximumAutomaticRedirections = HttpRequest.MaxAutoRedirectionCount;
}
foreach (KeyValuePair <string, string> header in http_request.Headers)
{
if (header.Value == null)
{
continue;
}
switch (header.Key)
{
case "User-Agent":
req.UserAgent = header.Value;
break;
case "Accept":
req.Accept = header.Value;
break;
case "Referer":
req.Referer = header.Value;
break;
case "Connection":
if (Regex.IsMatch(header.Value, "keep-alive", RegexOptions.IgnoreCase))
{
req.KeepAlive = true;
}
break;
case "Content-Type":
req.ContentType = header.Value;
break;
case "Expect":
req.Expect = header.Value;
break;
default:
req.Headers[header.Key] = header.Value;
break;
}
}
if (http_request.Method == HttpRequest.RequestMethod.POST)
{
req.Method = "POST";
if (http_request.PostData != null)
{
req.ContentLength = http_request.PostData.Length;
Stream req_stream = req.GetRequestStream();
req_stream.Write(http_request.PostData, 0, http_request.PostData.Length);
req_stream.Close();
}
}
HWResponse = (HttpWebResponse)req.GetResponse();
ResponseUrl = HWResponse.ResponseUri.ToString();
if (HWResponse.StatusCode == HttpStatusCode.Redirect)
{
web_routine_status = WebRoutineStatus.REDIRECTION;
throw new Exception("Redirected.\nURL:" + http_request.Url);
}
if (send_cookies)
{
lock (CookieContainer)
CookieContainer.Add(HWResponse.Cookies);
}
string accept;
if (http_request.Headers.TryGetValue("Accept", out accept) &&
accept == Properties.Web.Default.TextModeHttpRequestAcceptHeader &&
!Regex.IsMatch(HWResponse.ContentType, Properties.Web.Default.TextModeDownloadableContentTypePattern, RegexOptions.Compiled | RegexOptions.IgnoreCase)
)
{
web_routine_status = WebRoutineStatus.UNACCEPTABLE_CONTENT_TYPE;
throw new Exception("Unacceptable Content-Type:" + HWResponse.ContentType + "\nURL:" + http_request.Url);
}
MemoryStream result_ms = new MemoryStream();
int progress_max = (int)HWResponse.ContentLength;
res_stream = HWResponse.GetResponseStream();
byte[] buff = new byte[8192];
int total_byte_count = 0;
while (true)
{
int byte_count = res_stream.Read(buff, 0, buff.Length);
if (byte_count < 1)
{
break;
}
result_ms.Write(buff, 0, byte_count);
total_byte_count += byte_count;
show_progress(progress_max, total_byte_count);
if (Properties.Web.Default.MaxDownloadedFileLength > 0 &&
total_byte_count > Properties.Web.Default.MaxDownloadedFileLength
)
{
web_routine_status = WebRoutineStatus.FILE_TRUNCATED;
Log.Write("TRUNCATED. URL:" + http_request.Url);
break;
}
}
binary_result = result_ms.ToArray();
//if (res.StatusCode != HttpStatusCode.OK)
//{
// web_routine_status = WebRoutineStatus.DOWNLOAD_ERROR;
// string proxy = "";
// if (Proxy != null && Proxy.Address != null && Proxy.Address.Authority != null)
// proxy = Proxy.Address.Authority;
// Log.Error("Download error: " + res.StatusDescription + "\nURL:" + url + "\nPROXY: " + proxy);
// return false;
//}
if (web_routine_status == WebRoutineStatus.UNDEFINED)
{
web_routine_status = WebRoutineStatus.OK;
}
bool content_is_text = false;
if (Regex.IsMatch(HWResponse.ContentType, "text|json|xml", RegexOptions.Compiled | RegexOptions.IgnoreCase))
{
content_is_text = true;
}
CachedFile = Cache.CacheDownloadedFile(content_is_text, http_request.Url, http_request.PostString, ResponseUrl, BinaryResult, get_next_page_number(), cycle_identifier, web_routine_status);
return(true);
}
catch (ThreadAbortException)
{
}
catch (System.Net.WebException error)
{
string proxy = "";
if (Proxy != null && Proxy.WebProxy.Address != null && Proxy.WebProxy.Address.Authority != null)
{
proxy = Proxy.WebProxy.Address.Authority;
}
ErrorMessage = error.Message;
if (web_routine_status == WebRoutineStatus.UNDEFINED)
{
web_routine_status = WebRoutineStatus.DOWNLOAD_ERROR;
}
Log.Write("DOWNLOAD ERROR: " + error.Message + "\n" + error.StackTrace + "\nPROXY: " + proxy);
}
catch (Exception error)
{
ErrorMessage = error.Message;
if (web_routine_status == WebRoutineStatus.UNDEFINED)
{
web_routine_status = WebRoutineStatus.DOWNLOAD_ERROR;
}
Log.Error(error);
}
finally
{
if (res_stream != null)
{
res_stream.Close();
}
if (HWResponse != null)
{
HWResponse.Close();
}
}
if (ErrorMessage == null)
{
return(false);
}
return(false);
}
internal static void AddCookie(this CookieContainer cookieContainer, Uri uri, string cookieName, string cookieValue) =>
cookieContainer.SetCookies(uri, $"{cookieName}={cookieValue}");
static void Main(string[] args)
{
Console.WriteLine("----------------------Request 1 Start---------------------");
HttpWebRequest request1 = (HttpWebRequest)WebRequest.Create(Constants.UrlSeosprint);
request1.Method = Constants.MethodGet;
request1.Accept = Constants.AcceptGet;
request1.Host = Constants.DefaultHost;
request1.UserAgent = Constants.UserAgent;
request1.Proxy = null;
var response = (HttpWebResponse)request1.GetResponse();
HtmlDocument htmlDoc = helper.GetHtmlDocument(response);
string btnLoginRegExp = "(?<=class=\"btnlogin\" href=\")(.*)(?=\")";
var BtnIdQ1 = helper.GetStringByRegExp(htmlDoc, btnLoginRegExp);
var allCookies = response.Headers[HttpResponseHeader.SetCookie].Split(';').ToList();
Console.WriteLine("------------------------Query1Data---------------------------");
string UrlQ1 = string.Concat(Constants.UrlSeosprint, BtnIdQ1);
CookieContainer mainCookieContainer = new CookieContainer();
var uri = new Uri(Constants.UrlSeosprint);
mainCookieContainer.SetCookies(uri, allCookies[0].Trim());
mainCookieContainer.SetCookies(uri, allCookies[1].Trim());
string CookieQ1 = string.Empty;
foreach (var item in allCookies)
{
CookieQ1 += item + " ";
}
string RefererQ1 = string.Concat(Constants.UrlSeosprint, "/");
Console.WriteLine("Next Query Url:{0}\nCookies:{1}\nReferer:{2}\nBtnId:{3}", UrlQ1, CookieQ1, RefererQ1, BtnIdQ1);
Console.WriteLine("---------------------Request 2 Start---------------------");
HttpWebRequest request2 = (HttpWebRequest)WebRequest.Create(UrlQ1);
request2.Method = Constants.MethodGet;
request2.Accept = Constants.AcceptGet;
request2.Host = Constants.DefaultHost;
request2.UserAgent = Constants.UserAgent;
request2.KeepAlive = Constants.KeepAlive;
request2.Referer = RefererQ1;
request2.CookieContainer = mainCookieContainer;
var response2 = (HttpWebResponse)request2.GetResponse();
HtmlDocument htmlDoc2 = helper.GetHtmlDocument(response2);
var allCookies2 = response2.Headers[HttpResponseHeader.SetCookie].Split(';').ToList();
response2.Close();
var asklcntRegExp = "(?<=name=\"asklcnt\" value=\")(.*)(?=\")";
var asklcnt = helper.GetStringByRegExp(htmlDoc2, asklcntRegExp);
string LoginUrl = string.Concat(Constants.UrlSeosprint, "/proc-service/us-login.php");
//string CookieQ2 = CookieQ1 + allCookies2[0];
string RefererQ2 = UrlQ1;
Console.WriteLine("Login Url:{0}\nReferer:{1}\nasklcnt:{2}", LoginUrl, RefererQ2, asklcnt);
var CookieQ2 = string.Empty;
foreach (var item in allCookies2)
{
CookieQ2 += item + " ";
}
mainCookieContainer.Add(response2.Cookies[0]);
Console.WriteLine("Куки в контейнере созданном:{0}\nКуки котоыре пришли:{1}", mainCookieContainer.GetCookies(new Uri(Constants.UrlSeosprint))[3].Name, CookieQ2);
helper.LoadCapcha(RefererQ2, mainCookieContainer);
Console.WriteLine("---------------------Request 3 Start---------------------");
// HttpWebRequest request2 = (HttpWebRequest)WebRequest.Create(UrlQ1);
Console.ReadKey();
}
private void Send(string data, int redirections)
{
_data = data;
Encoding encode = Encoding.GetEncoding(_charset);
_headers.Remove("Content-Length");
if (!string.IsNullOrEmpty(data) && string.Compare(_method, "post", true) == 0)
{
_headers["Content-Length"] = string.Concat(Encoding.GetEncoding(_charset).GetBytes(data).Length);
if (string.IsNullOrEmpty(_headers["Content-Type"]))
{
_headers["Content-Type"] = "application/x-www-form-urlencoded; charset=" + _charset;
}
else if (_headers["Content-Type"].IndexOf("multipart/form-data") == -1)
{
if (_headers["Content-Type"].IndexOf("application/x-www-form-urlencoded") == -1)
{
_headers["Content-Type"] += "; application/x-www-form-urlencoded";
}
if (_headers["Content-Type"].IndexOf("charset=") == -1)
{
_headers["Content-Type"] += "; charset=" + _charset;
}
}
data += "\r\n\r\n";
}
Uri uri = new Uri(_action);
if (_cookieContainer != null)
{
CookieContainer cc = new CookieContainer();
if (_headers["Cookie"] != null)
{
cc.SetCookies(uri, _headers["Cookie"]);
}
Uri uri2 = new Uri(uri.AbsoluteUri.Insert(uri.Scheme.Length + 3, "httprequest."));
CookieCollection cookies = _cookieContainer.GetCookies(uri);
foreach (Cookie cookie in cookies)
{
cc.SetCookies(uri, string.Concat(cookie));
}
cookies = _cookieContainer.GetCookies(uri2);
foreach (Cookie cookie in cookies)
{
cc.SetCookies(uri, string.Concat(cookie));
}
_headers["Cookie"] = cc.GetCookieHeader(uri);
if (string.IsNullOrEmpty(_headers["Cookie"]))
{
_headers.Remove("Cookie");
}
}
_headers["Host"] = uri.Authority;
string http = _method + " " + uri.PathAndQuery + " HTTP/1.1\r\n";
foreach (string head in _headers)
{
http += head + ": " + _headers[head] + "\r\n";
}
http += "\r\n" + data;
_head = http;
if (_proxy != null)
{
HttpProxyRequest pr = new HttpProxyRequest();
pr.Method = _method;
pr.Action = _action;
pr.Charset = _charset;
pr.Head = _head;
pr.Data = data;
pr.Connection = _proxyConnection;
pr.Timeout = _timeout;
pr.MaximumAutomaticRedirections = _maximumAutomaticRedirections;
HttpProxyResponse response = _proxy.Send(pr);
Action = response.RequestAction;
_method = response.RequestMethod;
_headers = HttpRequest.ParseHttpRequestHeader(response.RequestHead);
_response = new HttpResponse(this, response.ResponseHead);
_response.SetStream(response.Response);
}
else
{
byte[] request = encode.GetBytes(http);
if (_client == null || _remote == null || string.Compare(_remote.Authority, uri.Authority, true) != 0)
{
_remote = uri;
this.Close();
_client = new TcpClient(uri.Host, uri.Port);
}
try {
_stream = getStream(uri);
_stream.Write(request, 0, request.Length);
} catch {
this.Close();
_client = new TcpClient(uri.Host, uri.Port);
_stream = getStream(uri);
_stream.Write(request, 0, request.Length);
}
receive(_stream, redirections, uri, encode);
}
}
/// <summary>
/// Send post data to server
/// </summary>
/// <param name="url"></param>
/// <param name="content"></param>
/// <param name="reqCcokies">Request Cookies</param>
/// <param name="resCookies">Response Cookies</param>
/// <returns>Return content</returns>
public string Login(string url, string content, CookieContainer reqCcokies, out CookieContainer resCookies)
{
string contentType = "application/x-www-form-urlencoded";
resCookies = new CookieContainer();
//contentType = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
int failedTimes = _tryTimes;
while (failedTimes-- > 0)
{
try
{
if (_delayTime > 0)
{
Thread.Sleep(_delayTime * 1000);
}
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(new Uri(url));
req.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322)";
byte[] buff = Encoding.GetEncoding(ChatSet).GetBytes(content);
req.KeepAlive = true;
req.Method = "POST";
req.Timeout = _timeout;
req.ContentType = contentType;
req.ContentLength = buff.Length;
req.AllowAutoRedirect = false;
if (null != _proxy && null != _proxy.Credentials)
{
req.UseDefaultCredentials = true;
}
else
{
req.Credentials = CredentialCache.DefaultCredentials;
}
if (reqCcokies != null)
{
//req.CookieContainer = new CookieContainer();
req.CookieContainer = (reqCcokies);
}
req.Proxy = _proxy;
req.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
//req.Connection = "Keep-Alive";
Stream reqStream = req.GetRequestStream();
reqStream.Write(buff, 0, buff.Length);
reqStream.Close();
//接收返回字串
HttpWebResponse res = (HttpWebResponse)req.GetResponse();
var co = res.GetResponseHeader("Set-Cookie");
if (!string.IsNullOrEmpty(co))
{
resCookies.SetCookies(new Uri(url), co);
}
StreamReader sr = new StreamReader(res.GetResponseStream(), Encoding.GetEncoding(ChatSet));
var result = sr.ReadToEnd();
try
{
res.Close();
//string _APSNET_SessionValue = res.Cookies["ASP.NET_SessionId"].Value;
}
catch (WebException ex)
{
throw ex;
}
return(result);
}
catch (Exception e)
{
LogFactory.ExceptionLog.Error("HTTP POST Error: " + e.Message);
LogFactory.ExceptionLog.Error("Url: " + url);
LogFactory.ExceptionLog.Error("Data: " + content);
}
}
return(string.Empty);
}
static HttpWebRequest Http_Core(string _type, string _url, Encoding _encoding, List <TItem> _header, object _conmand = null)
{
#region 启动HTTP请求之前的初始化操作
bool isget = false;
if (_type == "GET")
{
isget = true;
}
if (isget)
{
if (_conmand is List <TItem> )
{
List <TItem> _conmand_ = (List <TItem>)_conmand;
string param = "";
foreach (TItem item in _conmand_)
{
if (string.IsNullOrEmpty(param))
{
if (_url.Contains("?"))
{
param += "&" + item.Name + "=" + item.Value;
}
else
{
param = "?" + item.Name + "=" + item.Value;
}
}
else
{
param += "&" + item.Name + "=" + item.Value;
}
}
_url += param;
}
}
Uri uri = null;
try
{
uri = new Uri(_url);
}
catch { }
#endregion
if (uri != null)
{
//string _scheme = uri.Scheme.ToUpper();
try
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(uri);
req.Proxy = null;
req.Host = uri.Host;
req.Method = _type;
req.AllowAutoRedirect = false;
bool isContentType = true;
#region 设置请求头
if (_header != null && _header.Count > 0)
{
bool isnotHeader = true;
System.Collections.Specialized.NameValueCollection collection = null;
foreach (TItem item in _header)
{
string _Lower_Name = item.Name.ToLower();
switch (_Lower_Name)
{
case "host":
req.Host = item.Value;
break;
case "accept":
req.Accept = item.Value;
break;
case "user-agent":
req.UserAgent = item.Value;
break;
case "referer":
req.Referer = item.Value;
break;
case "content-type":
isContentType = false;
req.ContentType = item.Value;
break;
case "cookie":
#region 设置COOKIE
string _cookie = item.Value;
CookieContainer cookie_container = new CookieContainer();
if (_cookie.IndexOf(";") >= 0)
{
string[] arrCookie = _cookie.Split(';');
//加载Cookie
//cookie_container.SetCookies(new Uri(url), cookie);
foreach (string sCookie in arrCookie)
{
if (string.IsNullOrEmpty(sCookie))
{
continue;
}
if (sCookie.IndexOf("expires") > 0)
{
continue;
}
cookie_container.SetCookies(uri, sCookie);
}
}
else
{
cookie_container.SetCookies(uri, _cookie);
}
req.CookieContainer = cookie_container;
#endregion
break;
default:
if (isnotHeader && collection == null)
{
var property = typeof(WebHeaderCollection).GetProperty("InnerCollection", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic);
if (property != null)
{
collection = property.GetValue(req.Headers, null) as System.Collections.Specialized.NameValueCollection;
}
isnotHeader = false;
}
//设置对象的Header数据
if (collection != null)
{
collection[item.Name] = item.Value;
}
break;
}
}
}
#endregion
#region 设置POST数据
if (!isget)
{
if (_conmand != null)
{
if (_conmand is List <TItem> )
{
List <TItem> _conmand_ = (List <TItem>)_conmand;
//POST参数
if (isContentType)
{
req.ContentType = "application/x-www-form-urlencoded";
}
string param = "";
foreach (TItem item in _conmand_)
{
if (string.IsNullOrEmpty(param))
{
param = item.Name + "=" + item.Value;
}
else
{
param += "&" + item.Name + "=" + item.Value;
}
}
byte[] bs = _encoding.GetBytes(param);
req.ContentLength = bs.Length;
using (Stream reqStream = req.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
reqStream.Close();
}
}
else if (_conmand is string[])
{
try
{
string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
byte[] boundarybytes = Encoding.ASCII.GetBytes("\r\n--" + boundary + "\r\n");
byte[] endbytes = Encoding.ASCII.GetBytes("\r\n--" + boundary + "--\r\n");
req.ContentType = "multipart/form-data; boundary=" + boundary;
using (Stream reqStream = req.GetRequestStream())
{
string[] files = (string[])_conmand;
string headerTemplate = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\nContent-Type: {2}\r\n\r\n";
byte[] buff = new byte[1024];
for (int i = 0; i < files.Length; i++)
{
string file = files[i];
reqStream.Write(boundarybytes, 0, boundarybytes.Length);
string contentType = MimeMappingProvider.Shared.GetMimeMapping(file);
//string contentType = System.Web.MimeMapping.GetMimeMapping(file);
//string header = string.Format(headerTemplate, "file" + i, Path.GetFileName(file), contentType);
string header = string.Format(headerTemplate, "media", Path.GetFileName(file), contentType);//微信
byte[] headerbytes = _encoding.GetBytes(header);
reqStream.Write(headerbytes, 0, headerbytes.Length);
using (FileStream fileStream = new FileStream(file, FileMode.Open, FileAccess.Read))
{
int contentLen = fileStream.Read(buff, 0, buff.Length);
int Value = contentLen;
//文件上传开始
//tProgress.Invoke(new Action(() =>
//{
// tProgress.Maximum = (int)fileStream.Length;
// tProgress.Value = Value;
//}));
while (contentLen > 0)
{
//文件上传中
reqStream.Write(buff, 0, contentLen);
contentLen = fileStream.Read(buff, 0, buff.Length);
Value += contentLen;
//tProgress.Invoke(new Action(() =>
//{
// tProgress.Value = Value;
//}));
}
}
}
//文件上传结束
reqStream.Write(endbytes, 0, endbytes.Length);
}
}
catch
{
if (isContentType)
{
req.ContentType = null;
}
req.ContentLength = 0;
}
}
else
{
//POST参数
if (isContentType)
{
req.ContentType = "application/x-www-form-urlencoded";
}
string param = _conmand.ToString();
byte[] bs = _encoding.GetBytes(param);
req.ContentLength = bs.Length;
using (Stream reqStream = req.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
reqStream.Close();
}
}
}
else
{
req.ContentLength = 0;
}
}
#endregion
return(req);
}
catch
{
}
}
return(null);
}
/// <summary>
/// 创建POST方式Json数据的HTTP请求（包括了https站点请求）
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="parameters">随同请求POST的参数名称及参数值字典</param>
/// <param name="timeout">请求的超时时间</param>
/// <param name="userAgent">请求的客户端浏览器信息，可以为空</param>
/// <param name="requestEncoding">发送HTTP请求时所用的编码</param>
/// <param name="cookies">随同HTTP请求发送的Cookie信息，如果不需要身份验证可以为空</param>
/// <returns></returns>
public static HttpWebResponse CreatePostHttpResponseJson(string url, string postJson, string parameters, int?timeout, string userAgent, Encoding requestEncoding, string referer, ref CookieContainer M_Cookie, bool lbnLogin = false)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
if (requestEncoding == null)
{
throw new ArgumentNullException("requestEncoding");
}
HttpWebRequest request = null;
//如果是发送HTTPS请求
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request = WebRequest.Create(url) as HttpWebRequest;
request.ProtocolVersion = HttpVersion.Version10;
}
else
{
request = WebRequest.Create(url) as HttpWebRequest;
}
request.CookieContainer = M_Cookie;
// if (!(Cookie == null)) { request.CookieContainer.Add(Cookie); }
request.Method = "POST";
//服务端 判断 客户端 提交的是否是 JSON数据 时
if (!string.IsNullOrEmpty(postJson))
{
request.ContentType = "application/json;charset=UTF-8";
}
else
{
request.ContentType = "application/x-www-form-urlencoded";
}
request.KeepAlive = true;
if (!string.IsNullOrEmpty(userAgent))
{
request.UserAgent = userAgent;
}
else
{
request.UserAgent = DefaultUserAgent;
}
if (timeout.HasValue)
{
request.Timeout = timeout.Value;
}
//如果需要POST数据
#region post parameter 类似querystring格式
if (parameters != null)
{
byte[] data = requestEncoding.GetBytes(parameters);
using (Stream stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
stream.Close();
}
}
#endregion
#region post json
if (!string.IsNullOrEmpty(postJson))
{
using (var streamWriter = new StreamWriter(request.GetRequestStream()))
{
//string json = "{\"user\":\"test\"," +
// "\"password\":\"bla\"}";
streamWriter.Write(postJson);
streamWriter.Flush();
streamWriter.Close();
}
}
#endregion
if (!string.IsNullOrEmpty(referer))
{
request.Referer = referer;
}
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
if (request.CookieContainer != null)
{
response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);
if (lbnLogin)
{
string cookieheader = request.CookieContainer.GetCookieHeader(new Uri(url));
m_Cookie.SetCookies(new Uri(url), cookieheader);
m_Cookie = request.CookieContainer;
}
// string cookieheader = request.CookieContainer.GetCookieHeader(request.RequestUri);
// Cookie = response.Cookies;}
}
return(response);
}
public static string GetHtmlByBytes(string server, string URL, byte[] byteRequest, string[] cookie, out string header)
{
HttpWebRequest httpWebRequest;
HttpWebResponse webResponse;
httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(URL);
CookieContainer co = new CookieContainer();
string getString = " ";
foreach (string subCookie in cookie)
{
if (subCookie != null)
{
co.SetCookies(new Uri(server), subCookie);
}
}
httpWebRequest.Credentials = CredentialCache.DefaultCredentials;
httpWebRequest.CookieContainer = co;
httpWebRequest.ContentType = contentType;
httpWebRequest.Accept =
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9";
httpWebRequest.Referer = referer;
httpWebRequest.UserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.60";
httpWebRequest.Method = "Post";
httpWebRequest.Credentials = CredentialCache.DefaultCredentials;
httpWebRequest.ContentLength = byteRequest.Length;
Stream stream;
stream = httpWebRequest.GetRequestStream();
stream.Write(byteRequest, 0, byteRequest.Length);
stream.Close();
httpWebRequest.AllowAutoRedirect = false;
//Console.WriteLine(httpWebRequest.Headers.ToString());
//webResponse = httpWebRequest.GetResponse() as HttpWebResponse;
WebResponse res;
try
{
res = httpWebRequest.GetResponse();
}
catch (WebException e)
{
//if (e.Message.Contains("203"))
res = e.Response;
}
webResponse = (HttpWebResponse)res;
if (webResponse.StatusCode == HttpStatusCode.Found)
{
string strUrl = "";
strUrl = webResponse.Headers[HttpResponseHeader.Location];
if (!strUrl.Contains("vpns.jlu.edu.cn"))
{
strUrl = "https://vpns.jlu.edu.cn" + strUrl;
}
;
getString = GetHtml(strUrl, cookie, out header);
}
else
{
header = webResponse.Headers.ToString();
Stream responseStream = webResponse.GetResponseStream();
StreamReader streamReader = new StreamReader(responseStream);
getString = streamReader.ReadToEnd();
streamReader.Close();
responseStream.Close();
}
return(getString);
}
override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
var cookieUrl = new Uri(uri.Scheme + "://" + uri.Host); // don't include the path, Scheme is needed for mono compatibility
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(cookieUrl, c.Trim());
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
using (ClearanceHandler clearanceHandlr = new ClearanceHandler())
{
clearanceHandlr.MaxRetries = 30;
using (HttpClientHandler clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = webProxy,
UseProxy = (webProxy != null),
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
if (webRequest.EmulateBrowser == true)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
using (var request = new HttpRequestMessage())
{
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
using (response = await client.SendAsync(request))
{
var result = new WebClientByteResult
{
Content = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
IEnumerable <string> value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme)
{
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":");
}
else
{
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
}
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
}
}
}
}
}
void BeginSending()
{
isDone = false;
if (timeout > 0)
{
UniWeb.Instance.StartCoroutine(Timeout());
}
ThreadPool.QueueUserWorkItem(new WaitCallback(delegate(object t) {
try {
var retryCount = 0;
HttpConnection connection = null;
while (retryCount < maximumRedirects)
{
AddHeadersToRequest();
Uri pUri;
if (proxy != null)
{
pUri = proxy;
}
else
{
#if UNITY_WP8
pUri = uri;
#else
try {
if (System.Net.WebRequest.DefaultWebProxy != null)
{
pUri = System.Net.WebRequest.DefaultWebProxy.GetProxy(uri);
}
else
{
pUri = uri;
}
} catch (TypeInitializationException) {
//Could not get WebRequest type... default to no proxy.
pUri = uri;
}
#endif
}
connection = CreateConnection(pUri.Host, pUri.Port, pUri.Scheme.ToLower() == "https");
WriteToStream(connection.stream);
response = new Response(this);
try {
response.ReadFromStream(connection.stream);
} catch (HTTPException) {
retryCount++;
continue;
}
#if USE_KEEPALIVE
connectionPool[pUri.Host] = connection;
#endif
#if USE_COOKIES
if (enableCookies)
{
foreach (var i in response.headers.GetAll("Set-Cookie"))
{
try {
cookies.SetCookies(uri, i);
} catch (System.Net.CookieException) {
//Some cookies make the .NET cookie class barf. MEGH.
}
}
}
#endif
switch (response.status)
{
case 101:
upgradedConnection = connection;
break;
case 304:
break;
case 307:
uri = new Uri(response.headers.Get("Location"));
retryCount++;
continue;
case 302:
case 301:
method = "GET";
var location = response.headers.Get("Location");
if (location.StartsWith("/"))
{
uri = new Uri(uri, location);
}
else
{
uri = new Uri(location);
}
retryCount++;
continue;
default:
break;
}
break;
}
if (upgradedConnection == null)
{
#if USE_KEEPALIVE
if (response.protocol.ToUpper() == "HTTP/1.0" || response.headers.Get("Connection").ToUpper() == "CLOSE")
{
if (connectionPool.ContainsKey(uri.Host))
{
connectionPool.Remove(uri.Host);
}
connection.Dispose();
}
#else
connection.Dispose();
#endif
}
if (useCache && response != null)
{
var etag = response.headers.Get("etag");
if (etag.Length > 0)
{
etags [uri.AbsoluteUri] = etag;
}
}
} catch (Exception e) {
exception = e;
response = null;
}
isDone = true;
}));
}
private RequestState <TResponse> SendWebRequest <TResponse>(string httpMethod, string absoluteUrl, object request,
Action <TResponse> onSuccess, Action <TResponse, Exception> onError)
{
if (httpMethod == null)
{
throw new ArgumentNullException("httpMethod");
}
var requestUri = absoluteUrl;
var httpGetOrDeleteOrHead = (httpMethod == "GET" || httpMethod == "DELETE" || httpMethod == "HEAD");
var hasQueryString = request != null && httpGetOrDeleteOrHead;
if (hasQueryString)
{
var queryString = QueryStringSerializer.SerializeToString(request);
if (!string.IsNullOrEmpty(queryString))
{
requestUri += "?" + queryString;
}
}
#if SILVERLIGHT && !WINDOWS_PHONE && !NETFX_CORE
var creator = this.UseBrowserHttpHandling
? System.Net.Browser.WebRequestCreator.BrowserHttp
: System.Net.Browser.WebRequestCreator.ClientHttp;
var webRequest = (HttpWebRequest)creator.Create(new Uri(requestUri));
if (StoreCookies && !UseBrowserHttpHandling)
{
if (ShareCookiesWithBrowser)
{
if (CookieContainer == null)
{
CookieContainer = new CookieContainer();
}
CookieContainer.SetCookies(new Uri(BaseUri), System.Windows.Browser.HtmlPage.Document.Cookies);
}
webRequest.CookieContainer = CookieContainer;
}
#else
_webRequest = (HttpWebRequest)WebRequest.Create(requestUri);
if (StoreCookies)
{
_webRequest.CookieContainer = CookieContainer;
}
#endif
#if !SILVERLIGHT
if (!DisableAutoCompression)
{
_webRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
_webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
}
#endif
var requestState = new RequestState <TResponse>
{
HttpMethod = httpMethod,
Url = requestUri,
#if SILVERLIGHT && !WINDOWS_PHONE && !NETFX_CORE
WebRequest = webRequest,
#else
WebRequest = _webRequest,
#endif
Request = request,
OnSuccess = onSuccess,
OnError = onError,
#if SILVERLIGHT
HandleCallbackOnUIThread = HandleCallbackOnUIThread,
#endif
};
requestState.StartTimer(this.Timeout.GetValueOrDefault(DefaultTimeout));
#if SILVERLIGHT && !WINDOWS_PHONE && !NETFX_CORE
SendWebRequestAsync(httpMethod, request, requestState, webRequest);
#else
SendWebRequestAsync(httpMethod, request, requestState, _webRequest);
#endif
return(requestState);
}
static void Main(string[] args)
{
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(CurrentDomain_UnhandledException);
System.Reflection.Assembly asm = System.Reflection.Assembly.GetExecutingAssembly();
WindowsPrincipal pricipal = new WindowsPrincipal(WindowsIdentity.GetCurrent());
bool hasAdministrativeRight = pricipal.IsInRole(WindowsBuiltInRole.Administrator);
if (!hasAdministrativeRight)
{
string parameter = string.Concat(args);
RunElevated(asm.Location, parameter);
Process.GetCurrentProcess().Kill();
}
Console.WriteLine("[+] Checking initial configuration...");
Int16 levels = 3;
Int16.TryParse(ConfigurationManager.AppSettings["levels"], out levels);
Int16 connCount = 30;
Int16.TryParse(ConfigurationManager.AppSettings["count"], out connCount);
Int16 duration = 300;
Int16.TryParse(ConfigurationManager.AppSettings["duration"], out duration);
if (duration < 180)
{
duration = 180;
}
if (duration > 3600)
{
duration = 3600;
}
ClientType type = ClientType.VU;
try
{
switch (ConfigurationManager.AppSettings["type"].ToLower())
{
case "sbu":
type = ClientType.SBU;
break;
default:
type = ClientType.VU;
break;
}
}
catch { }
Uri site = null;
try
{
site = new Uri(ConfigurationManager.AppSettings["uri"]);
}
catch (Exception ex)
{
Console.WriteLine("URI not valid");
return;
}
Dictionary <String, String> headers = new Dictionary <string, string>();
if (!String.IsNullOrWhiteSpace(ConfigurationManager.AppSettings["Cookie"]))
{
try
{
CookieContainer tmp = new CookieContainer();
tmp.SetCookies(site, ConfigurationManager.AppSettings["Cookie"]);
headers.Add("Cookie", ConfigurationManager.AppSettings["Cookie"]);
}
catch (Exception ex)
{
Console.WriteLine("Error parsing cookie");
return;
}
}
try
{
if (!String.IsNullOrWhiteSpace(ConfigurationManager.AppSettings["User-Agent"]))
{
headers.Add("User-Agent", ConfigurationManager.AppSettings["User-Agent"]);
}
}
catch (Exception ex)
{
Console.WriteLine("Error parsing User-Agent");
return;
}
Console.WriteLine("[+] Checking zabbix agents...");
Console.WriteLine("[+] Checking date and time from NTP servers...");
try
{
DateTime ntpdate = NTP.GetNetworkUTCTime();
Console.WriteLine(" NTP UTC data: " + ntpdate.ToString(Thread.CurrentThread.CurrentCulture));
TimeSpan ts = ntpdate - DateTime.UtcNow;
if (Math.Abs(ts.TotalSeconds) > 60)
{
Console.WriteLine(" Updating local time");
Console.WriteLine(" Old time: " + DateTime.Now.ToString(Thread.CurrentThread.CurrentCulture));
NTP.SetSystemTime(ntpdate);
Console.WriteLine(" New time: " + DateTime.Now.ToString(Thread.CurrentThread.CurrentCulture));
}
else
{
Console.WriteLine(" Local time is up to date");
}
}
catch (Exception ex)
{
Console.WriteLine("[!] Error updating local time: " + ex.Message);
}
List <ZabbixConfig> zbxConfig = new List <ZabbixConfig>();
ZabbixConfigSection ZabbixManagers = (ZabbixConfigSection)ConfigurationManager.GetSection("zabbixMonitors");
if (ZabbixManagers != null)
{
foreach (ZabbixConfigElement zbxHost in ZabbixManagers.ZabbixConfigElements)
{
//Realiza teste de conex'ao em cada um dos zabbix listados
try
{
Console.Write("[*] Zabbix agent on " + zbxHost.Host + ":" + zbxHost.Port);
using (Zabbix zbx = new Zabbix(zbxHost.Host, zbxHost.Port))
{
String tst = zbx.GetItem("system.hostname");
}
zbxConfig.Add(new ZabbixConfig(zbxHost.Name, zbxHost.Host, zbxHost.Port));
Console.WriteLine("\t\tOK");
}
catch {
Console.WriteLine("\t\tError");
Console.WriteLine("Error Getting information from Zabbix " + zbxHost.Name + " (" + zbxHost.Host + ":" + zbxHost.Port + ")");
return;
}
}
}
Profile prof = null;
/*
* if (args.Length > 4)
* {
* try
* {
* FileInfo profileFile = new FileInfo(args[4]);
* if (profileFile.Extension == ".prof")
* {
* prof = new Profile();
* prof.LoadProfile(profileFile.FullName);
*
* if ((prof.BaseUri == null) || (prof.Uris.Count == 0))
* prof = null;
* }
* }
* catch(Exception ex) {
* prof = null;
* }
* }*/
IPEndPoint proxy = null;// new IPEndPoint(IPAddress.Parse("10.0.10.1"), 80);
if (!String.IsNullOrEmpty((string)ConfigurationManager.AppSettings["proxy"]))
{
Uri tProxy = null;
try
{
tProxy = new Uri(ConfigurationManager.AppSettings["proxy"]);
proxy = new IPEndPoint(IPAddress.Parse(tProxy.Host), tProxy.Port);
}
catch (Exception ex)
{
Console.WriteLine("Proxy not valid. Proxy should be an IP URI. Ex: http://10.10.10.10:3389");
return;
}
}
Int32 sleepTime = 0;
try
{
Int32.TryParse((string)ConfigurationManager.AppSettings["sleeptime"], out sleepTime);
}
catch { }
TestBuilder builder = new TestBuilder();
/*
* if (prof != null)
* builder.Fetch(prof);
* else
* builder.Fetch(
* site,
* proxy,
* 1);
*/
//
Console.WriteLine("[+] Building test environment...");
builder.Fetch(
site,
proxy,
levels,
headers);
TestEnvironment env = builder.Build(proxy, type, 5);
env.HTTPHeaders = headers;
env.ZabbixMonitors = zbxConfig;
env.SleepTime = sleepTime;
env.VirtualUsers = connCount;
env.ConnectionString = new DbConnectionString(ConfigurationManager.ConnectionStrings["LoadTest"]);
env.Start();
Console.WriteLine("[+] Starting test...");
Console.WriteLine("[+] System started (" + (prof != null ? "prof" : "scan") + ")!");
Console.WriteLine("[+] Total test duration: " + duration + " seconds");
Console.WriteLine("[!] Press CTRL + C to finish the teste and generate report");
_running = true;
_tmpEnd = new Timer(new TimerCallback(tmpEnd), null, duration * 1000, duration * 1000);
Console.CancelKeyPress += new ConsoleCancelEventHandler(myHandler);
Console.WriteLine("");
DateTime dStart = DateTime.Now;
while (_running)
{
System.Threading.Thread.Sleep(500);
TimeSpan ts = DateTime.Now - dStart;
Console.Write("\r");
Console.Write("Test Duration: {0}", ts.ToString(@"hh\:mm\:ss"));
}
ClearCurrentConsoleLine();
Console.WriteLine("");
env.Stop();
/*
* TestEnvironment env = new TestEnvironment();
* env.LoadConfig("temp.env");*/
//Gera o relatório
Console.WriteLine("[+] Building report...");
env.BuildReports();
//env.DropDatabase();
}
public static string GetHtmlContent(string url, int userAgent = 0, Dictionary <string, string> headers = null)
{
try
{
var cookies = new CookieContainer();
var handler = new HttpClientHandler {
CookieContainer = cookies
};
using (var myHttpWebRequest = new HttpClient(handler)
{
Timeout = new TimeSpan(0, 0, 15)
})
{
myHttpWebRequest.DefaultRequestHeaders.Add("Method", "GET");
myHttpWebRequest.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
switch (userAgent)
{
case 1:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/13F69 MicroMessenger/6.3.16 NetType/WIFI Language/zh_CN");
break;
case 2:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1");
break;
case 3:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; NOKIA; Lumia 930) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.10586");
break;
case 4:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "NativeHost");
break;
case 5:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Dalvik/1.6.0 (Linux; U; Android 4.4.2; NoxW Build/KOT49H) ITV_5.7.1.46583");
break;
case 6:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "qqlive");
break;
case 7:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Dalvik/1.6.0 (Linux; U; Android 4.2.2; 6S Build/JDQ39E)");
break;
case 8:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) XIAMI-MUSIC/3.0.2 Chrome/51.0.2704.106 Electron/1.2.8 Safari/537.36");
break;
case 9:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Dalvik/1.6.0 (Linux; U; Android 4.4.2; MI 6 Build/NMF26X)");
break;
default:
myHttpWebRequest.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
break;
}
if (headers != null)
{
foreach (var k in headers)
{
if (k.Key == "Cookie")
{
cookies.SetCookies(new Uri(url), k.Value.Replace(";", ","));
}
else
{
myHttpWebRequest.DefaultRequestHeaders.Add(k.Key, k.Value);
}
}
}
return(myHttpWebRequest.GetStringAsync(url).Result);
}
}
catch (Exception ex)
{
AddLog(ex.ToString());
return(null);
}
}
async Task <HttpResponseMessage> DoProcessRequest(HttpRequestMessage request, URL javaUrl, HttpURLConnection httpConnection, CancellationToken cancellationToken, RequestRedirectionState redirectState)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"{this}.DoProcessRequest ()");
}
if (cancellationToken.IsCancellationRequested)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, " cancelled");
}
cancellationToken.ThrowIfCancellationRequested();
}
CancellationTokenSource waitHandleSource = new CancellationTokenSource();
try {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $" connecting");
}
CancellationToken linkedToken = CancellationTokenSource.CreateLinkedTokenSource(waitHandleSource.Token, cancellationToken).Token;
await Task.WhenAny(
httpConnection.ConnectAsync(),
Task.Run(() => {
linkedToken.WaitHandle.WaitOne();
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Wait handle task finished");
}
}))
.ConfigureAwait(false);
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $" connected");
}
} catch (Java.Net.ConnectException ex) {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Connection exception {ex}");
}
// Wrap it nicely in a "standard" exception so that it's compatible with HttpClientHandler
throw new WebException(ex.Message, ex, WebExceptionStatus.ConnectFailure, null);
} finally{
//If not already cancelled, cancel the WaitOne through the waitHandleSource to prevent an orphaned thread
waitHandleSource.Cancel();
}
if (cancellationToken.IsCancellationRequested)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, " cancelled");
}
await DisconnectAsync(httpConnection).ConfigureAwait(continueOnCapturedContext: false);
cancellationToken.ThrowIfCancellationRequested();
}
CancellationTokenRegistration cancelRegistration = default(CancellationTokenRegistration);
HttpStatusCode statusCode = HttpStatusCode.OK;
Uri connectionUri = null;
try {
cancelRegistration = cancellationToken.Register(() => {
DisconnectAsync(httpConnection).ContinueWith(t => {
if (t.Exception != null)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Disconnection exception: {t.Exception}");
}
}, TaskScheduler.Default);
}, useSynchronizationContext: false);
if (httpConnection.DoOutput)
{
using (var stream = await request.Content.ReadAsStreamAsync().ConfigureAwait(false)) {
await stream.CopyToAsync(httpConnection.OutputStream, 4096, cancellationToken)
.ConfigureAwait(false);
}
}
statusCode = await Task.Run(() => (HttpStatusCode)httpConnection.ResponseCode).ConfigureAwait(false);
connectionUri = new Uri(httpConnection.URL.ToString());
} finally {
cancelRegistration.Dispose();
}
if (cancellationToken.IsCancellationRequested)
{
await DisconnectAsync(httpConnection).ConfigureAwait(continueOnCapturedContext: false);
cancellationToken.ThrowIfCancellationRequested();
}
// If the request was redirected we need to put the new URL in the request
request.RequestUri = connectionUri;
var ret = new AndroidHttpResponseMessage(javaUrl, httpConnection)
{
RequestMessage = request,
ReasonPhrase = httpConnection.ResponseMessage,
StatusCode = statusCode,
};
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Status code: {statusCode}");
}
bool disposeRet;
if (HandleRedirect(statusCode, httpConnection, redirectState, out disposeRet))
{
if (disposeRet)
{
ret.Dispose();
ret = null;
}
return(ret);
}
switch (statusCode)
{
case HttpStatusCode.Unauthorized:
case HttpStatusCode.ProxyAuthenticationRequired:
// We don't resend the request since that would require new set of credentials if the
// ones provided in Credentials are invalid (or null) and that, in turn, may require asking the
// user which is not something that should be taken care of by us and in this
// context. The application should be responsible for this.
// HttpClientHandler throws an exception in this instance, but I think it's not a good
// idea. We'll return the response message with all the information required by the
// application to fill in the blanks and provide the requested credentials instead.
//
// We return the body of the response too, but the Java client will throw
// a FileNotFound exception if we attempt to access the input stream.
// Instead we try to read the error stream and return an default message if the error stream isn't readable.
ret.Content = GetErrorContent(httpConnection, new StringContent("Unauthorized", Encoding.ASCII));
CopyHeaders(httpConnection, ret);
if (ret.Headers.WwwAuthenticate != null)
{
ProxyAuthenticationRequested = false;
CollectAuthInfo(ret.Headers.WwwAuthenticate);
}
else if (ret.Headers.ProxyAuthenticate != null)
{
ProxyAuthenticationRequested = true;
CollectAuthInfo(ret.Headers.ProxyAuthenticate);
}
ret.RequestedAuthentication = RequestedAuthentication;
return(ret);
}
if (!IsErrorStatusCode(statusCode))
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Reading...");
}
ret.Content = GetContent(httpConnection, httpConnection.InputStream);
}
else
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Status code is {statusCode}, reading...");
}
// For 400 >= response code <= 599 the Java client throws the FileNotFound exception when attempting to read from the input stream.
// Instead we try to read the error stream and return an empty string if the error stream isn't readable.
ret.Content = GetErrorContent(httpConnection, new StringContent(String.Empty, Encoding.ASCII));
}
CopyHeaders(httpConnection, ret);
IEnumerable <string> cookieHeaderValue;
if (!UseCookies || CookieContainer == null || !ret.Headers.TryGetValues("Set-Cookie", out cookieHeaderValue) || cookieHeaderValue == null)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"No cookies");
}
return(ret);
}
try {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Parsing cookies");
}
CookieContainer.SetCookies(connectionUri, String.Join(",", cookieHeaderValue));
} catch (Exception ex) {
// We don't want to terminate the response because of a bad cookie, hence just reporting
// the issue. We might consider adding a virtual method to let the user handle the
// issue, but not sure if it's really needed. Set-Cookie header will be part of the
// header collection so the user can always examine it if they spot an error.
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Failed to parse cookies in the server response. {ex.GetType ()}: {ex.Message}");
}
}
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Returning");
}
return(ret);
}
public static void SetCookies_InvalidInput_Throws()
{
CookieContainer cc = new CookieContainer();
Assert.Throws<ArgumentNullException>(() => cc.SetCookies(null, "")); // Null uri
Assert.Throws<ArgumentNullException>(() => cc.SetCookies(u5, null)); // Null header
}
public string get_response(string url, string method, string post, string cookie)
{
string str_return = "";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
if (!String.IsNullOrEmpty(post))
{
byte[] send = Encoding.ASCII.GetBytes(post);
request.ContentLength = send.Length;
request.ContentType = "application/x-www-form-urlencoded";
request.KeepAlive = true;
Stream stream = request.GetRequestStream();
stream.Write(send, 0, send.Length);
stream.Close();
}
if (!String.IsNullOrEmpty(cookie))
{
CookieContainer container = new CookieContainer();
container.SetCookies(new Uri(url), cookie);
request.CookieContainer = container;
}
this.txt_request.Text = get_request_header(request);
try
{
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.Default);
str_return = reader.ReadToEnd();
return str_return;
}
catch (WebException error)
{
return error.Message;
}
}