public void CookieHeaderToDictionaryNull()
{
// null cookie header
var expectedCookieDictionary = new Dictionary <string, string>();
CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(null));
}
protected override IDictionary <string, string> GetCookies()
{
if (Settings.ExtraFieldData.TryGetValue("cookie", out var cookies))
{
return(CookieUtil.CookieHeaderToDictionary((string)cookies));
}
return(base.GetCookies());
}
public void CookieHeaderToDictionaryMalformed()
{
// malformed cookies
var cookieHeader = "__cfduidd6237f041586694295; __cf_;bm TlOng; good_cookie=value";
var expectedCookieDictionary = new Dictionary <string, string> {
{ "good_cookie", "value" },
};
CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(cookieHeader));
}
public void CookieHeaderToDictionaryGood()
{
// valid cookies with non-alpha characters in the value
var cookieHeader = "__cfduid=d6237f041586694295; __cf_bm=TlOng/xyqckk-TMen38z+0RFYA7YA=";
var expectedCookieDictionary = new Dictionary <string, string>
{
{ "__cfduid", "d6237f041586694295" }, { "__cf_bm", "TlOng/xyqckk-TMen38z+0RFYA7YA=" }
};
CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(cookieHeader));
}
public void CookieHeaderToDictionaryDuplicateKeys()
{
// cookie with duplicate keys and whitespace separator instead of ;
// this cookie is not valid according to the standard, but it occurs in Jackett because we are concatenating
// cookies in many parts of the code (and we are not doing it well). this is safe because the whitespace
// can't be part of the key nor the value.
var cookieHeader = "__cfduid=d6237f041586694295; __cf_bm=TlOng/xyqckk-TMen38z+0RFYA7YA= __cf_bm=test";
var expectedCookieDictionary = new Dictionary <string, string>
{
{ "__cfduid", "d6237f041586694295" },
{ "__cf_bm", "test" } // we always assume the latest value is the most recent
};
CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(cookieHeader));
}
protected override async Task <WebResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrWhiteSpace(webRequest.Cookies))
{
// don't include the path, Scheme is needed for mono compatibility
var requestUri = new Uri(webRequest.Url);
var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host);
var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies);
foreach (var kv in cookieDictionary)
{
cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value));
}
}
var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion();
using (var clearanceHandlr = new ClearanceHandler(userAgent))
{
clearanceHandlr.MaxTries = 10;
using (var clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = webProxy,
UseProxy = (webProxy != null),
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
if (webRequest.EmulateBrowser == true)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
using (var request = new HttpRequestMessage())
{
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
using (response = await client.SendAsync(request))
{
var result = new WebResult
{
ContentBytes = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
var value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter WebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = int.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
// URL decoding apparently is needed to, without it e.g. Demonoid download is broken
// TODO: is it always needed (not just for relative redirects)?
var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme)
{
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":");
}
else
{
newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
}
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
}
}
}
}
}
protected override IDictionary <string, string> GetCookies()
{
return(CookieUtil.CookieHeaderToDictionary("mam_id=" + Settings.MamId));
}
protected override IDictionary <string, string> GetCookies()
{
return(CookieUtil.CookieHeaderToDictionary(Settings.Cookie));
}
public async Task <Captcha> GetConfigurationForSetup(bool automaticlogin)
{
var login = _definition.Login;
if (login == null || login.Method != "form")
{
return(null);
}
var loginUrl = ResolvePath(login.Path);
Cookies = null;
if (login.Cookies != null)
{
Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies));
}
var requestBuilder = new HttpRequestBuilder(loginUrl.AbsoluteUri)
{
LogResponseContent = true,
Method = HttpMethod.Get,
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
Cookies = landingResult.GetCookies();
// Some sites have a temporary redirect before the login page, we need to process it.
//if (_definition.Followredirect)
//{
// await FollowIfRedirect(landingResult, loginUrl.AbsoluteUri, overrideCookies: landingResult.Cookies, accumulateCookies: true);
//}
var htmlParser = new HtmlParser();
landingResultDocument = htmlParser.ParseDocument(landingResult.Content);
Captcha captcha = null;
if (login.Captcha != null)
{
captcha = await GetCaptcha(login);
}
if (captcha != null && automaticlogin)
{
_logger.Error("CardigannIndexer ({0}): Found captcha during automatic login, aborting", _definition.Id);
}
return(captcha);
}
public async Task DoLogin()
{
var login = _definition.Login;
if (login.Method == "post")
{
var pairs = new Dictionary <string, string>();
foreach (var input in login.Inputs)
{
var value = ApplyGoTemplateText(input.Value);
pairs.Add(input.Key, value);
}
var loginUrl = ResolvePath(login.Path).ToString();
CookiesUpdater(null, null);
var requestBuilder = new HttpRequestBuilder(loginUrl)
{
LogResponseContent = true,
Method = HttpMethod.Post,
AllowAutoRedirect = true,
SuppressHttpError = true,
Encoding = _encoding
};
foreach (var pair in pairs)
{
requestBuilder.AddFormParameter(pair.Key, pair.Value);
}
requestBuilder.Headers.Add("Referer", SiteLink);
var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
Cookies = response.GetCookies();
CheckForError(response, login.Error);
CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30));
}
else if (login.Method == "form")
{
var loginUrl = ResolvePath(login.Path).ToString();
var queryCollection = new NameValueCollection();
var pairs = new Dictionary <string, string>();
var formSelector = login.Form;
if (formSelector == null)
{
formSelector = "form";
}
// landingResultDocument might not be initiated if the login is caused by a relogin during a query
if (landingResultDocument == null)
{
await GetConfigurationForSetup(true);
}
var form = landingResultDocument.QuerySelector(formSelector);
if (form == null)
{
throw new CardigannConfigException(_definition, string.Format("Login failed: No form found on {0} using form selector {1}", loginUrl, formSelector));
}
var inputs = form.QuerySelectorAll("input");
if (inputs == null)
{
throw new CardigannConfigException(_definition, string.Format("Login failed: No inputs found on {0} using form selector {1}", loginUrl, formSelector));
}
var submitUrlstr = form.GetAttribute("action");
if (login.Submitpath != null)
{
submitUrlstr = login.Submitpath;
}
foreach (var input in inputs)
{
var name = input.GetAttribute("name");
if (name == null)
{
continue;
}
var value = input.GetAttribute("value");
if (value == null)
{
value = "";
}
pairs[name] = value;
}
foreach (var input in login.Inputs)
{
var value = ApplyGoTemplateText(input.Value);
var inputKey = input.Key;
if (login.Selectors)
{
var inputElement = landingResultDocument.QuerySelector(input.Key);
if (inputElement == null)
{
throw new CardigannConfigException(_definition, string.Format("Login failed: No input found using selector {0}", input.Key));
}
inputKey = inputElement.GetAttribute("name");
}
pairs[inputKey] = value;
}
// selector inputs
if (login.Selectorinputs != null)
{
foreach (var selectorinput in login.Selectorinputs)
{
string value = null;
try
{
value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild);
pairs[selectorinput.Key] = value;
}
catch (Exception ex)
{
throw new CardigannException(string.Format("Error while parsing selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message));
}
}
}
// getselector inputs
if (login.Getselectorinputs != null)
{
foreach (var selectorinput in login.Getselectorinputs)
{
string value = null;
try
{
value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild);
queryCollection[selectorinput.Key] = value;
}
catch (Exception ex)
{
throw new CardigannException(string.Format("Error while parsing get selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message));
}
}
}
if (queryCollection.Count > 0)
{
submitUrlstr += "?" + queryCollection.GetQueryString();
}
var submitUrl = ResolvePath(submitUrlstr, new Uri(loginUrl));
// automatically solve simpleCaptchas, if used
var simpleCaptchaPresent = landingResultDocument.QuerySelector("script[src*=\"simpleCaptcha\"]");
if (simpleCaptchaPresent != null)
{
var captchaUrl = ResolvePath("simpleCaptcha.php?numImages=1");
var requestBuilder = new HttpRequestBuilder(captchaUrl.ToString())
{
LogResponseContent = true,
Method = HttpMethod.Get,
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", loginUrl);
var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
pairs["captchaSelection"] = captchaSelection;
pairs["submitme"] = "X";
}
if (login.Captcha != null)
{
var captcha = login.Captcha;
Settings.ExtraFieldData.TryGetValue("CAPTCHA", out var captchaText);
if (captchaText != null)
{
var input = captcha.Input;
if (login.Selectors)
{
var inputElement = landingResultDocument.QuerySelector(captcha.Input);
if (inputElement == null)
{
throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input));
}
input = inputElement.GetAttribute("name");
}
pairs[input] = (string)captchaText;
}
}
// clear landingResults/Document, otherwise we might use an old version for a new relogin (if GetConfigurationForSetup() wasn't called before)
landingResult = null;
landingResultDocument = null;
HttpResponse loginResult = null;
var enctype = form.GetAttribute("enctype");
if (enctype == "multipart/form-data")
{
var headers = new Dictionary <string, string>();
var boundary = "---------------------------" + DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds.ToString().Replace(".", "");
var bodyParts = new List <string>();
foreach (var pair in pairs)
{
var part = "--" + boundary + "\r\n" +
"Content-Disposition: form-data; name=\"" + pair.Key + "\"\r\n" +
"\r\n" +
pair.Value;
bodyParts.Add(part);
}
bodyParts.Add("--" + boundary + "--");
headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary);
var body = string.Join("\r\n", bodyParts);
var requestBuilder = new HttpRequestBuilder(submitUrl.ToString())
{
LogResponseContent = true,
Method = HttpMethod.Post,
AllowAutoRedirect = true,
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
requestBuilder.SetCookies(Cookies);
foreach (var pair in pairs)
{
requestBuilder.AddFormParameter(pair.Key, pair.Value);
}
foreach (var header in headers)
{
requestBuilder.SetHeader(header.Key, header.Value);
}
var request = requestBuilder.Build();
request.SetContent(body);
loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
}
else
{
var requestBuilder = new HttpRequestBuilder(submitUrl.ToString())
{
LogResponseContent = true,
Method = HttpMethod.Post,
AllowAutoRedirect = true,
SuppressHttpError = true,
Encoding = _encoding
};
requestBuilder.SetCookies(Cookies);
requestBuilder.Headers.Add("Referer", loginUrl);
foreach (var pair in pairs)
{
requestBuilder.AddFormParameter(pair.Key, pair.Value);
}
loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
}
Cookies = loginResult.GetCookies();
CheckForError(loginResult, login.Error);
CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30));
}
else if (login.Method == "cookie")
{
CookiesUpdater(null, null);
Settings.ExtraFieldData.TryGetValue("cookie", out var cookies);
CookiesUpdater(CookieUtil.CookieHeaderToDictionary((string)cookies), DateTime.Now + TimeSpan.FromDays(30));
}
else if (login.Method == "get")
{
var queryCollection = new NameValueCollection();
foreach (var input in login.Inputs)
{
var value = ApplyGoTemplateText(input.Value);
queryCollection.Add(input.Key, value);
}
var loginUrl = ResolvePath(login.Path + "?" + queryCollection.GetQueryString()).ToString();
CookiesUpdater(null, null);
var requestBuilder = new HttpRequestBuilder(loginUrl)
{
LogResponseContent = true,
Method = HttpMethod.Get,
SuppressHttpError = true,
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
Cookies = response.GetCookies();
CheckForError(response, login.Error);
CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30));
}
else if (login.Method == "oneurl")
{
var oneUrl = ApplyGoTemplateText(login.Inputs["oneurl"]);
var loginUrl = ResolvePath(login.Path + oneUrl).ToString();
CookiesUpdater(null, null);
var requestBuilder = new HttpRequestBuilder(loginUrl)
{
LogResponseContent = true,
Method = HttpMethod.Get,
SuppressHttpError = true,
Encoding = _encoding
};
requestBuilder.Headers.Add("Referer", SiteLink);
var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
Cookies = response.GetCookies();
CheckForError(response, login.Error);
CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30));
}
else
{
throw new NotImplementedException("Login method " + login.Method + " not implemented");
}
}
