// Wrapper for Run which takes care of CloudFlare challenges
private async Task <WebClientByteResult> RunCloudFlare(WebRequest request)
{
WebClientByteResult result = await Run(request);
// check if we've received a CloudFlare challenge
if (result.Status == HttpStatusCode.ServiceUnavailable && ((request.Cookies != null && request.Cookies.Contains("__cfduid")) || result.Cookies.Contains("__cfduid")))
{
logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge");
// solve the challenge
string pageContent = Encoding.UTF8.GetString(result.Content);
Uri uri = new Uri(request.Url);
string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri);
logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri));
// wait...
await Task.Delay(5000);
// request clearanceUri to get cf_clearance cookie
var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer);
logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies));
// add new cf_clearance cookies to the original request
request.Cookies = response.Cookies + request.Cookies;
// re-run the original request with updated cf_clearance cookie
result = await Run(request);
// add cf_clearance cookie to the final result so we update the config for the next request
result.Cookies = response.Cookies + " " + result.Cookies;
}
return(result);
}
public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified()
{
var res = new WebClientByteResult()
{
RedirectingTo = "list?p=1"
};
var req = new WebRequest()
{
Url = "http://my.domain.com/page.php"
};
// Not fully qualified requiring redirect
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1");
// Fully qualified not needing modified
res.RedirectingTo = "http://a.domain/page.htm";
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm");
// Relative requiring redirect
req.Url = "http://my.domain.com/dir/page.php";
res.RedirectingTo = "a/dir/page.html";
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html");
}
// Wrapper for Run which takes care of CloudFlare challenges, calls RunCurl
override protected async Task <WebClientByteResult> Run(WebRequest request)
{
WebClientByteResult result = await RunCurl(request);
// check if we've received a CloudFlare challenge
string[] server;
if (result.Status == HttpStatusCode.ServiceUnavailable && result.Headers.TryGetValue("server", out server) && server[0] == "cloudflare-nginx")
{
logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge");
// solve the challenge
string pageContent = Encoding.UTF8.GetString(result.Content);
Uri uri = new Uri(request.Url);
string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri);
logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri));
// wait...
await Task.Delay(5000);
// request clearanceUri to get cf_clearance cookie
var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer);
logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies));
// add new cf_clearance cookies to the original request
request.Cookies = response.Cookies + request.Cookies;
// re-run the original request with updated cf_clearance cookie
result = await RunCurl(request);
// add cf_clearance cookie to the final result so we update the config for the next request
result.Cookies = response.Cookies + " " + result.Cookies;
}
return(result);
}
private async Task <WebClientByteResult> Run(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
}
else
{
if (!string.IsNullOrEmpty(request.RawBody))
{
logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
}
var result = new WebClientByteResult()
{
Content = response.Content,
Cookies = response.Cookies,
Status = response.Status
};
if (response.HeaderList != null)
{
foreach (var header in response.HeaderList)
{
switch (header[0].ToLowerInvariant())
{
case "location":
result.RedirectingTo = header[1];
break;
}
}
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return(result);
}
public async Task<WebClientByteResult> GetBytes(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url));
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
}
else
{
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
}
var result = new WebClientByteResult()
{
Content = response.Content,
Cookies = response.CookieHeader,
Status = response.Status
};
if (response.Headers != null)
{
foreach(var header in response.Headers)
{
if(string.Equals(header.Key, "location", StringComparison.InvariantCultureIgnoreCase) && header.Value !=null)
{
result.RedirectingTo = header.Value;
}
}
}
logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status));
return result;
}
private async Task<WebClientByteResult> Run(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
}
else
{
if (!string.IsNullOrEmpty(request.RawBody))
{
logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
}
var result = new WebClientByteResult()
{
Content = response.Content,
Cookies = response.Cookies,
Status = response.Status
};
if (response.HeaderList != null)
{
foreach (var header in response.HeaderList)
{
switch (header[0].ToLowerInvariant())
{
case "location":
result.RedirectingTo = header[1];
break;
case "refresh":
if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable)
{
//"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
var redirval = "";
var value = header[1];
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
result.Status = System.Net.HttpStatusCode.Redirect;
var redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
break;
}
}
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return result;
}
private async Task<WebClientByteResult> Run(WebRequest request)
{
var args = new StringBuilder();
args.AppendFormat("--url \"{0}\" ", request.Url);
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
if (!string.IsNullOrWhiteSpace(request.Cookies))
{
args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
}
if (!string.IsNullOrWhiteSpace(request.Referer))
{
args.AppendFormat("--referer \"{0}\" ", request.Referer);
}
if (request.PostData != null && request.PostData.Count > 0)
{
var postString = new FormUrlEncodedContent(request.PostData).ReadAsStringAsync().Result;
args.AppendFormat("--data \"{0}\" ", postString);
}
var tempFile = Path.GetTempFileName();
args.AppendFormat("--output \"{0}\" ", tempFile);
if (Startup.DoSSLFix == true)
{
// http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
// https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
args.Append("--cipher " + SSLFix.CipherList);
}
string stdout = null;
await Task.Run(() =>
{
stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix?"curl":"curl.exe", args.ToString(), true);
});
var outputData = File.ReadAllBytes(tempFile);
File.Delete(tempFile);
stdout = Encoding.UTF8.GetString(outputData);
var result = new WebClientByteResult();
var headSplit = stdout.IndexOf("\r\n\r\n");
if (headSplit < 0)
throw new Exception("Invalid response");
var headers = stdout.Substring(0, headSplit);
var headerCount = 0;
foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
if (headerCount == 0)
{
var responseCode = int.Parse(header.Split(' ')[1]);
result.Status = (HttpStatusCode)responseCode;
}
else
{
var headerSplitIndex = header.IndexOf(':');
if (headerSplitIndex > 0)
{
var name = header.Substring(0, headerSplitIndex).ToLowerInvariant();
var value = header.Substring(headerSplitIndex + 1);
switch (name)
{
case "set-cookie":
var cookieDataSplit = value.IndexOf(';');
if (cookieDataSplit > 0)
{
result.Cookies += value.Substring(0, cookieDataSplit + 1) + " ";
}//Location
break;
case "location":
result.RedirectingTo = value.Trim();
break;
}
}
}
headerCount++;
}
result.Content = new byte[outputData.Length - (headSplit + 3)];
var dest = 0;
for (int i= headSplit+4;i< outputData.Length; i++)
{
result.Content[dest] = outputData[i];
dest++;
}
logger.Debug("WebClientByteResult returned " + result.Status);
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return result;
}
private async Task<WebClientByteResult> Run(WebRequest request)
{
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(request.Cookies))
{
var uri = new Uri(request.Url);
foreach (var c in request.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var client = new HttpClient(new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
});
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
HttpResponseMessage response = null;
if (request.Type == RequestType.POST)
{
var content = new FormUrlEncodedContent(request.PostData);
response = await client.PostAsync(request.Url, content);
}
else
{
response = await client.GetAsync(request.Url);
}
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable<string> cookieHeaders;
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
var cookieBuilder = new StringBuilder();
foreach (var c in cookieHeaders)
{
cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';')+1));
}
result.Cookies = cookieBuilder.ToString().TrimEnd();
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return result;
}
override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
{
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.EmulateBrowser == true)
{
request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent);
}
else
{
request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion());
}
// clear cookies from cookiecontainer
var oldCookies = cookies.GetCookies(request.RequestUri);
foreach (Cookie oldCookie in oldCookies)
{
oldCookie.Expired = true;
}
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
// add cookies to cookiecontainer
var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host); // don't include the path, Scheme is needed for mono compatibility
foreach (var ccookiestr in webRequest.Cookies.Split(';'))
{
var cookiestrparts = ccookiestr.Split('=');
var name = cookiestrparts[0].Trim();
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
var value = "";
if (cookiestrparts.Length >= 2)
{
value = cookiestrparts[1].Trim();
}
var cookie = new Cookie(name, value);
cookies.Add(cookieUrl, cookie);
}
}
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
foreach (var header in response.Headers)
{
IEnumerable <string> value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
logger.Debug(value);
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
private async Task<WebClientByteResult> Run(WebRequest webRequest)
{
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var client = new HttpClient(new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
});
if(webRequest.EmulateBrowser)
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
else
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.Add(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
request.Content = new StringContent(webRequest.RawBody);
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable<string> cookieHeaders;
var responseCookies = new List<Tuple<string, string>>();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return result;
}
private async Task<WebClientByteResult> Run(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
}
else
{
if (request.PostData != null && request.PostData.Count() > 0)
{
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
}
var result = new WebClientByteResult()
{
Content = response.Content,
Cookies = response.Cookies,
Status = response.Status
};
if (response.HeaderList != null)
{
foreach (var header in response.HeaderList)
{
switch (header[0].ToLowerInvariant())
{
case "location":
result.RedirectingTo = header[1];
break;
}
}
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return result;
}
private async Task <WebClientByteResult> Run(WebRequest request)
{
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(request.Cookies))
{
var uri = new Uri(request.Url);
foreach (var c in request.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var client = new HttpClient(new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
});
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
HttpResponseMessage response = null;
if (request.Type == RequestType.POST)
{
var content = new FormUrlEncodedContent(request.PostData);
response = await client.PostAsync(request.Url, content);
}
else
{
response = await client.GetAsync(request.Url);
}
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
var cookieBuilder = new StringBuilder();
foreach (var c in cookieHeaders)
{
cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1));
}
result.Cookies = cookieBuilder.ToString().TrimEnd();
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return(result);
}
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
// Follow up to 5 redirects
for (int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
break;
await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies);
}
}
override protected async Task <WebClientByteResult> Run(WebRequest request)
{
var args = new StringBuilder();
if (Startup.ProxyConnection != null)
{
args.AppendFormat("-x " + Startup.ProxyConnection + " ");
}
args.AppendFormat("--url \"{0}\" ", request.Url);
if (request.EmulateBrowser == true)
{
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
}
else
{
args.AppendFormat("-i -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());
}
if (!string.IsNullOrWhiteSpace(request.Cookies))
{
args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
}
if (!string.IsNullOrWhiteSpace(request.Referer))
{
args.AppendFormat("--referer \"{0}\" ", request.Referer);
}
if (!string.IsNullOrEmpty(request.RawBody))
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", postString);
}
var tempFile = Path.GetTempFileName();
args.AppendFormat("--output \"{0}\" ", tempFile);
if (Startup.DoSSLFix == true)
{
// http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
// https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
args.Append("--cipher " + SSLFix.CipherList);
}
if (Startup.IgnoreSslErrors == true)
{
args.Append("-k ");
}
args.Append("-H \"Accept-Language: en-US,en\" ");
args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" ");
string stdout = null;
await Task.Run(() =>
{
stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
});
var outputData = File.ReadAllBytes(tempFile);
File.Delete(tempFile);
stdout = Encoding.UTF8.GetString(outputData);
var result = new WebClientByteResult();
var headSplit = stdout.IndexOf("\r\n\r\n");
if (headSplit < 0)
{
throw new Exception("Invalid response");
}
var headers = stdout.Substring(0, headSplit);
if (Startup.ProxyConnection != null)
{
// the proxy provided headers too so we need to split headers again
var headSplit1 = stdout.IndexOf("\r\n\r\n", headSplit + 4);
if (headSplit1 > 0)
{
headers = stdout.Substring(headSplit + 4, headSplit1 - (headSplit + 4));
headSplit = headSplit1;
}
}
var headerCount = 0;
var cookieBuilder = new StringBuilder();
var cookies = new List <Tuple <string, string> >();
foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
if (headerCount == 0)
{
var responseCode = int.Parse(header.Split(' ')[1]);
result.Status = (HttpStatusCode)responseCode;
}
else
{
var headerSplitIndex = header.IndexOf(':');
if (headerSplitIndex > 0)
{
var name = header.Substring(0, headerSplitIndex).ToLowerInvariant();
var value = header.Substring(headerSplitIndex + 1);
switch (name)
{
case "set-cookie":
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
cookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
break;
case "location":
result.RedirectingTo = value.Trim();
break;
case "refresh":
//"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
var redirval = "";
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
result.Status = System.Net.HttpStatusCode.Redirect;
var redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
break;
}
}
}
headerCount++;
}
foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
result.Content = new byte[outputData.Length - (headSplit + 3)];
var dest = 0;
for (int i = headSplit + 4; i < outputData.Length; i++)
{
result.Content[dest] = outputData[i];
dest++;
}
logger.Debug("WebClientByteResult returned " + result.Status);
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return(result);
}
protected async Task <WebClientByteResult> RunCurl(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer, request.Headers);
}
else
{
if (!string.IsNullOrEmpty(request.RawBody))
{
logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.Headers, request.RawBody);
}
var result = new WebClientByteResult()
{
Content = response.Content,
Cookies = response.Cookies,
Status = response.Status
};
if (response.HeaderList != null)
{
foreach (var header in response.HeaderList)
{
var key = header[0].ToLowerInvariant();
result.Headers[key] = new string[] { header[1] }; // doesn't support multiple identical headers?
switch (key)
{
case "location":
result.RedirectingTo = header[1];
break;
case "refresh":
if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable)
{
//"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
var redirval = "";
var value = header[1];
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
result.Status = System.Net.HttpStatusCode.Redirect;
var redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
break;
}
}
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return(result);
}
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
if (incomingResponse.IsRedirect)
{
var redirRequestCookies = "";
if (accumulateCookies)
{
redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : ""));
} else
{
redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
}
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,
Cookies = redirRequestCookies
});
Mapper.Map(redirectedResponse, incomingResponse);
}
}
protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
{
// Follow up to 5 redirects
for (int i = 0; i < 5; i++)
{
if (!response.IsRedirect)
break;
await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
if (accumulateCookies)
{
CookieHeader = ResolveCookies((CookieHeader != null && CookieHeader != ""? CookieHeader + " " : "") + (overrideCookies != null && overrideCookies != "" ? overrideCookies + " " : "") + response.Cookies);
overrideCookies = response.Cookies = CookieHeader;
}
if (overrideCookies != null && response.Cookies == null)
{
response.Cookies = overrideCookies;
}
}
}
private async Task<WebClientByteResult> Run(WebRequest request)
{
var args = new StringBuilder();
if (Startup.ProxyConnection != null)
{
args.AppendFormat("-x " + Startup.ProxyConnection + " ");
}
args.AppendFormat("--url \"{0}\" ", request.Url);
if (request.EmulateBrowser)
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
else
args.AppendFormat("-i -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());
if (!string.IsNullOrWhiteSpace(request.Cookies))
{
args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
}
if (!string.IsNullOrWhiteSpace(request.Referer))
{
args.AppendFormat("--referer \"{0}\" ", request.Referer);
}
if (!string.IsNullOrEmpty(request.RawBody))
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
} else if (request.PostData != null && request.PostData.Count() > 0)
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", postString);
}
var tempFile = Path.GetTempFileName();
args.AppendFormat("--output \"{0}\" ", tempFile);
if (Startup.DoSSLFix == true)
{
// http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
// https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
args.Append("--cipher " + SSLFix.CipherList);
}
if (Startup.IgnoreSslErrors == true)
{
args.Append("-k ");
}
args.Append("-H \"Accept-Language: en-US,en\" ");
args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" ");
string stdout = null;
await Task.Run(() =>
{
stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString() , true);
});
var outputData = File.ReadAllBytes(tempFile);
File.Delete(tempFile);
stdout = Encoding.UTF8.GetString(outputData);
var result = new WebClientByteResult();
var headSplit = stdout.IndexOf("\r\n\r\n");
if (headSplit < 0)
throw new Exception("Invalid response");
var headers = stdout.Substring(0, headSplit);
if (Startup.ProxyConnection != null)
{
// the proxy provided headers too so we need to split headers again
var headSplit1 = stdout.IndexOf("\r\n\r\n",headSplit + 4);
if (headSplit1 > 0)
{
headers = stdout.Substring(headSplit + 4,headSplit1 - (headSplit + 4));
headSplit = headSplit1;
}
}
var headerCount = 0;
var cookieBuilder = new StringBuilder();
var cookies = new List<Tuple<string, string>>();
foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
if (headerCount == 0)
{
var responseCode = int.Parse(header.Split(' ')[1]);
result.Status = (HttpStatusCode)responseCode;
}
else
{
var headerSplitIndex = header.IndexOf(':');
if (headerSplitIndex > 0)
{
var name = header.Substring(0, headerSplitIndex).ToLowerInvariant();
var value = header.Substring(headerSplitIndex + 1);
switch (name)
{
case "set-cookie":
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
cookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
break;
case "location":
result.RedirectingTo = value.Trim();
break;
case "refresh":
//"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
var redirval = "";
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
result.Status = System.Net.HttpStatusCode.Redirect;
var redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
break;
}
}
}
headerCount++;
}
foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
result.Content = new byte[outputData.Length - (headSplit + 3)];
var dest = 0;
for (int i = headSplit + 4; i < outputData.Length; i++)
{
result.Content[dest] = outputData[i];
dest++;
}
logger.Debug("WebClientByteResult returned " + result.Status);
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return result;
}
private async Task <WebClientByteResult> Run(WebRequest webRequest)
{
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var client = new HttpClient(new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
});
if (webRequest.EmulateBrowser)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.Add(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
private async Task<WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var useProxy = false;
WebProxy proxyServer = null;
if (Startup.ProxyConnection != null)
{
proxyServer = new WebProxy(Startup.ProxyConnection, false);
useProxy = true;
}
var client = new HttpClient(new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = proxyServer,
UseProxy = useProxy
});
if (webRequest.EmulateBrowser)
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
else
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.Add(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
request.Content = new StringContent(webRequest.RawBody);
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable<string> cookieHeaders;
var responseCookies = new List<Tuple<string, string>>();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return result;
}
private async Task <WebClientByteResult> Run(WebRequest request)
{
var args = new StringBuilder();
args.AppendFormat("--url \"{0}\" ", request.Url);
if (request.EmulateBrowser)
{
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
}
else
{
args.AppendFormat("-i -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());
}
if (!string.IsNullOrWhiteSpace(request.Cookies))
{
args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
}
if (!string.IsNullOrWhiteSpace(request.Referer))
{
args.AppendFormat("--referer \"{0}\" ", request.Referer);
}
if (!string.IsNullOrEmpty(request.RawBody))
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", postString);
}
var tempFile = Path.GetTempFileName();
args.AppendFormat("--output \"{0}\" ", tempFile);
if (Startup.DoSSLFix == true)
{
// http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
// https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
args.Append("--cipher " + SSLFix.CipherList);
}
string stdout = null;
await Task.Run(() =>
{
stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
});
var outputData = File.ReadAllBytes(tempFile);
File.Delete(tempFile);
stdout = Encoding.UTF8.GetString(outputData);
var result = new WebClientByteResult();
var headSplit = stdout.IndexOf("\r\n\r\n");
if (headSplit < 0)
{
throw new Exception("Invalid response");
}
var headers = stdout.Substring(0, headSplit);
var headerCount = 0;
var cookieBuilder = new StringBuilder();
var cookies = new List <Tuple <string, string> >();
foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
if (headerCount == 0)
{
var responseCode = int.Parse(header.Split(' ')[1]);
result.Status = (HttpStatusCode)responseCode;
}
else
{
var headerSplitIndex = header.IndexOf(':');
if (headerSplitIndex > 0)
{
var name = header.Substring(0, headerSplitIndex).ToLowerInvariant();
var value = header.Substring(headerSplitIndex + 1);
switch (name)
{
case "set-cookie":
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
cookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
break;
case "location":
result.RedirectingTo = value.Trim();
break;
}
}
}
headerCount++;
}
foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
result.Content = new byte[outputData.Length - (headSplit + 3)];
var dest = 0;
for (int i = headSplit + 4; i < outputData.Length; i++)
{
result.Content[dest] = outputData[i];
dest++;
}
logger.Debug("WebClientByteResult returned " + result.Status);
ServerUtil.ResureRedirectIsFullyQualified(request, result);
return(result);
}
protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
var byteResult = new WebClientByteResult();
// Map to byte
Mapper.Map(response, byteResult);
await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies);
// Map to string
Mapper.Map(byteResult, response);
}
private async Task <WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(uri, c);
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var useProxy = false;
WebProxy proxyServer = null;
if (Startup.ProxyConnection != null)
{
proxyServer = new WebProxy(Startup.ProxyConnection, false);
useProxy = true;
}
ClearanceHandler clearanceHandlr = new ClearanceHandler();
HttpClientHandler clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = proxyServer,
UseProxy = useProxy
};
clearanceHandlr.InnerHandler = clientHandlr;
var client = new HttpClient(clearanceHandlr);
if (webRequest.EmulateBrowser)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.Add(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';') + 1))));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
{
if (incomingResponse.IsRedirect)
{
// Do redirect
var redirectedResponse = await webclient.GetBytes(new WebRequest()
{
Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
Referer = referrer,
Cookies = overrideCookies ?? CookieHeader
});
Mapper.Map(redirectedResponse, incomingResponse);
}
}
override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
var cookieUrl = new Uri(uri.Scheme + "://" + uri.Host); // don't include the path, Scheme is needed for mono compatibility
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(cookieUrl, c.Trim());
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var useProxy = false;
using (ClearanceHandler clearanceHandlr = new ClearanceHandler())
{
IWebProxy proxyServer = null;
var proxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(proxyUrl))
{
useProxy = true;
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = await Dns.GetHostAddressesAsync(serverConfig.ProxyUrl);
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
proxyServer = new SocksWebProxy(socksConfig, false);
}
else
{
proxyServer = new WebProxy(proxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
}
using (HttpClientHandler clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = proxyServer,
UseProxy = useProxy,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
if (webRequest.EmulateBrowser == true)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
using (var request = new HttpRequestMessage())
{
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
using (response = await client.SendAsync(request))
{
var result = new WebClientByteResult
{
Content = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
IEnumerable <string> value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
}
}
}
}
}