public void CookieHeaderToDictionaryNull() { // null cookie header var expectedCookieDictionary = new Dictionary <string, string>(); CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(null)); }
protected override IDictionary <string, string> GetCookies() { if (Settings.ExtraFieldData.TryGetValue("cookie", out var cookies)) { return(CookieUtil.CookieHeaderToDictionary((string)cookies)); } return(base.GetCookies()); }
public void CookieHeaderToDictionaryMalformed() { // malformed cookies var cookieHeader = "__cfduidd6237f041586694295; __cf_;bm TlOng; good_cookie=value"; var expectedCookieDictionary = new Dictionary <string, string> { { "good_cookie", "value" }, }; CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(cookieHeader)); }
public void CookieHeaderToDictionaryGood() { // valid cookies with non-alpha characters in the value var cookieHeader = "__cfduid=d6237f041586694295; __cf_bm=TlOng/xyqckk-TMen38z+0RFYA7YA="; var expectedCookieDictionary = new Dictionary <string, string> { { "__cfduid", "d6237f041586694295" }, { "__cf_bm", "TlOng/xyqckk-TMen38z+0RFYA7YA=" } }; CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(cookieHeader)); }
public void CookieHeaderToDictionaryDuplicateKeys() { // cookie with duplicate keys and whitespace separator instead of ; // this cookie is not valid according to the standard, but it occurs in Jackett because we are concatenating // cookies in many parts of the code (and we are not doing it well). this is safe because the whitespace // can't be part of the key nor the value. var cookieHeader = "__cfduid=d6237f041586694295; __cf_bm=TlOng/xyqckk-TMen38z+0RFYA7YA= __cf_bm=test"; var expectedCookieDictionary = new Dictionary <string, string> { { "__cfduid", "d6237f041586694295" }, { "__cf_bm", "test" } // we always assume the latest value is the most recent }; CollectionAssert.AreEqual(expectedCookieDictionary, CookieUtil.CookieHeaderToDictionary(cookieHeader)); }
protected override async Task <WebResult> Run(WebRequest webRequest) { ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; var cookies = new CookieContainer(); if (!string.IsNullOrWhiteSpace(webRequest.Cookies)) { // don't include the path, Scheme is needed for mono compatibility var requestUri = new Uri(webRequest.Url); var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host); var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies); foreach (var kv in cookieDictionary) { cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value)); } } var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion(); using (var clearanceHandlr = new ClearanceHandler(userAgent)) { clearanceHandlr.MaxTries = 10; using (var clientHandlr = new HttpClientHandler { CookieContainer = cookies, AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. UseCookies = true, Proxy = webProxy, UseProxy = (webProxy != null), AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }) { clearanceHandlr.InnerHandler = clientHandlr; using (var client = new HttpClient(clearanceHandlr)) { if (webRequest.EmulateBrowser == true) { client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); } else { client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion()); } HttpResponseMessage response = null; using (var request = new HttpRequestMessage()) { request.Headers.ExpectContinue = false; request.RequestUri = new Uri(webRequest.Url); if (webRequest.Headers != null) { foreach (var header in webRequest.Headers) { if (header.Key != "Content-Type") { request.Headers.TryAddWithoutValidation(header.Key, header.Value); } } } if (!string.IsNullOrEmpty(webRequest.Referer)) { request.Headers.Referrer = new Uri(webRequest.Referer); } if (!string.IsNullOrEmpty(webRequest.RawBody)) { var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault(); if (type.HasValue) { var str = new StringContent(webRequest.RawBody); str.Headers.Remove("Content-Type"); str.Headers.Add("Content-Type", type.Value.Value); request.Content = str; } else { request.Content = new StringContent(webRequest.RawBody); } request.Method = HttpMethod.Post; } else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) { request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); } request.Method = HttpMethod.Post; } else { request.Method = HttpMethod.Get; } using (response = await client.SendAsync(request)) { var result = new WebResult { ContentBytes = await response.Content.ReadAsByteArrayAsync() }; foreach (var header in response.Headers) { var value = header.Value; result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); } // some cloudflare clients are using a refresh header // Pull it out manually if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) { var refreshHeaders = response.Headers.GetValues("Refresh"); var redirval = ""; var redirtime = 0; if (refreshHeaders != null) { foreach (var value in refreshHeaders) { var start = value.IndexOf("="); var end = value.IndexOf(";"); var len = value.Length; if (start > -1) { redirval = value.Substring(start + 1); result.RedirectingTo = redirval; // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature // of this cloudflare approach..don't want to alter WebResult.IsRedirect because normally // it shoudln't include service unavailable..only if we have this redirect header. response.StatusCode = System.Net.HttpStatusCode.Redirect; redirtime = int.Parse(value.Substring(0, end)); System.Threading.Thread.Sleep(redirtime * 1000); } } } } if (response.Headers.Location != null) { result.RedirectingTo = response.Headers.Location.ToString(); } // Mono won't add the baseurl to relative redirects. // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" // See issue #1200 if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) { // URL decoding apparently is needed to, without it e.g. Demonoid download is broken // TODO: is it always needed (not just for relative redirects)? var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme) { newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":"); } else { newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); } logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); result.RedirectingTo = newRedirectingTo; } result.Status = response.StatusCode; // Compatiblity issue between the cookie format and httpclient // Pull it out manually ignoring the expiry date then set it manually // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer var responseCookies = new List <Tuple <string, string> >(); if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders)) { foreach (var value in cookieHeaders) { var nameSplit = value.IndexOf('='); if (nameSplit > -1) { responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); } } var cookieBuilder = new StringBuilder(); foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) { cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); } result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); return(result); } } } } } }
protected override IDictionary <string, string> GetCookies() { return(CookieUtil.CookieHeaderToDictionary("mam_id=" + Settings.MamId)); }
protected override IDictionary <string, string> GetCookies() { return(CookieUtil.CookieHeaderToDictionary(Settings.Cookie)); }
public async Task <Captcha> GetConfigurationForSetup(bool automaticlogin) { var login = _definition.Login; if (login == null || login.Method != "form") { return(null); } var loginUrl = ResolvePath(login.Path); Cookies = null; if (login.Cookies != null) { Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies)); } var requestBuilder = new HttpRequestBuilder(loginUrl.AbsoluteUri) { LogResponseContent = true, Method = HttpMethod.Get, Encoding = _encoding }; requestBuilder.Headers.Add("Referer", SiteLink); if (Cookies != null) { requestBuilder.SetCookies(Cookies); } var request = requestBuilder.Build(); landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition); Cookies = landingResult.GetCookies(); // Some sites have a temporary redirect before the login page, we need to process it. //if (_definition.Followredirect) //{ // await FollowIfRedirect(landingResult, loginUrl.AbsoluteUri, overrideCookies: landingResult.Cookies, accumulateCookies: true); //} var htmlParser = new HtmlParser(); landingResultDocument = htmlParser.ParseDocument(landingResult.Content); Captcha captcha = null; if (login.Captcha != null) { captcha = await GetCaptcha(login); } if (captcha != null && automaticlogin) { _logger.Error("CardigannIndexer ({0}): Found captcha during automatic login, aborting", _definition.Id); } return(captcha); }
public async Task DoLogin() { var login = _definition.Login; if (login.Method == "post") { var pairs = new Dictionary <string, string>(); foreach (var input in login.Inputs) { var value = ApplyGoTemplateText(input.Value); pairs.Add(input.Key, value); } var loginUrl = ResolvePath(login.Path).ToString(); CookiesUpdater(null, null); var requestBuilder = new HttpRequestBuilder(loginUrl) { LogResponseContent = true, Method = HttpMethod.Post, AllowAutoRedirect = true, SuppressHttpError = true, Encoding = _encoding }; foreach (var pair in pairs) { requestBuilder.AddFormParameter(pair.Key, pair.Value); } requestBuilder.Headers.Add("Referer", SiteLink); var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); Cookies = response.GetCookies(); CheckForError(response, login.Error); CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); } else if (login.Method == "form") { var loginUrl = ResolvePath(login.Path).ToString(); var queryCollection = new NameValueCollection(); var pairs = new Dictionary <string, string>(); var formSelector = login.Form; if (formSelector == null) { formSelector = "form"; } // landingResultDocument might not be initiated if the login is caused by a relogin during a query if (landingResultDocument == null) { await GetConfigurationForSetup(true); } var form = landingResultDocument.QuerySelector(formSelector); if (form == null) { throw new CardigannConfigException(_definition, string.Format("Login failed: No form found on {0} using form selector {1}", loginUrl, formSelector)); } var inputs = form.QuerySelectorAll("input"); if (inputs == null) { throw new CardigannConfigException(_definition, string.Format("Login failed: No inputs found on {0} using form selector {1}", loginUrl, formSelector)); } var submitUrlstr = form.GetAttribute("action"); if (login.Submitpath != null) { submitUrlstr = login.Submitpath; } foreach (var input in inputs) { var name = input.GetAttribute("name"); if (name == null) { continue; } var value = input.GetAttribute("value"); if (value == null) { value = ""; } pairs[name] = value; } foreach (var input in login.Inputs) { var value = ApplyGoTemplateText(input.Value); var inputKey = input.Key; if (login.Selectors) { var inputElement = landingResultDocument.QuerySelector(input.Key); if (inputElement == null) { throw new CardigannConfigException(_definition, string.Format("Login failed: No input found using selector {0}", input.Key)); } inputKey = inputElement.GetAttribute("name"); } pairs[inputKey] = value; } // selector inputs if (login.Selectorinputs != null) { foreach (var selectorinput in login.Selectorinputs) { string value = null; try { value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild); pairs[selectorinput.Key] = value; } catch (Exception ex) { throw new CardigannException(string.Format("Error while parsing selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message)); } } } // getselector inputs if (login.Getselectorinputs != null) { foreach (var selectorinput in login.Getselectorinputs) { string value = null; try { value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild); queryCollection[selectorinput.Key] = value; } catch (Exception ex) { throw new CardigannException(string.Format("Error while parsing get selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message)); } } } if (queryCollection.Count > 0) { submitUrlstr += "?" + queryCollection.GetQueryString(); } var submitUrl = ResolvePath(submitUrlstr, new Uri(loginUrl)); // automatically solve simpleCaptchas, if used var simpleCaptchaPresent = landingResultDocument.QuerySelector("script[src*=\"simpleCaptcha\"]"); if (simpleCaptchaPresent != null) { var captchaUrl = ResolvePath("simpleCaptcha.php?numImages=1"); var requestBuilder = new HttpRequestBuilder(captchaUrl.ToString()) { LogResponseContent = true, Method = HttpMethod.Get, Encoding = _encoding }; requestBuilder.Headers.Add("Referer", loginUrl); var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content); var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString(); pairs["captchaSelection"] = captchaSelection; pairs["submitme"] = "X"; } if (login.Captcha != null) { var captcha = login.Captcha; Settings.ExtraFieldData.TryGetValue("CAPTCHA", out var captchaText); if (captchaText != null) { var input = captcha.Input; if (login.Selectors) { var inputElement = landingResultDocument.QuerySelector(captcha.Input); if (inputElement == null) { throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input)); } input = inputElement.GetAttribute("name"); } pairs[input] = (string)captchaText; } } // clear landingResults/Document, otherwise we might use an old version for a new relogin (if GetConfigurationForSetup() wasn't called before) landingResult = null; landingResultDocument = null; HttpResponse loginResult = null; var enctype = form.GetAttribute("enctype"); if (enctype == "multipart/form-data") { var headers = new Dictionary <string, string>(); var boundary = "---------------------------" + DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds.ToString().Replace(".", ""); var bodyParts = new List <string>(); foreach (var pair in pairs) { var part = "--" + boundary + "\r\n" + "Content-Disposition: form-data; name=\"" + pair.Key + "\"\r\n" + "\r\n" + pair.Value; bodyParts.Add(part); } bodyParts.Add("--" + boundary + "--"); headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary); var body = string.Join("\r\n", bodyParts); var requestBuilder = new HttpRequestBuilder(submitUrl.ToString()) { LogResponseContent = true, Method = HttpMethod.Post, AllowAutoRedirect = true, Encoding = _encoding }; requestBuilder.Headers.Add("Referer", SiteLink); requestBuilder.SetCookies(Cookies); foreach (var pair in pairs) { requestBuilder.AddFormParameter(pair.Key, pair.Value); } foreach (var header in headers) { requestBuilder.SetHeader(header.Key, header.Value); } var request = requestBuilder.Build(); request.SetContent(body); loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition); } else { var requestBuilder = new HttpRequestBuilder(submitUrl.ToString()) { LogResponseContent = true, Method = HttpMethod.Post, AllowAutoRedirect = true, SuppressHttpError = true, Encoding = _encoding }; requestBuilder.SetCookies(Cookies); requestBuilder.Headers.Add("Referer", loginUrl); foreach (var pair in pairs) { requestBuilder.AddFormParameter(pair.Key, pair.Value); } loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); } Cookies = loginResult.GetCookies(); CheckForError(loginResult, login.Error); CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); } else if (login.Method == "cookie") { CookiesUpdater(null, null); Settings.ExtraFieldData.TryGetValue("cookie", out var cookies); CookiesUpdater(CookieUtil.CookieHeaderToDictionary((string)cookies), DateTime.Now + TimeSpan.FromDays(30)); } else if (login.Method == "get") { var queryCollection = new NameValueCollection(); foreach (var input in login.Inputs) { var value = ApplyGoTemplateText(input.Value); queryCollection.Add(input.Key, value); } var loginUrl = ResolvePath(login.Path + "?" + queryCollection.GetQueryString()).ToString(); CookiesUpdater(null, null); var requestBuilder = new HttpRequestBuilder(loginUrl) { LogResponseContent = true, Method = HttpMethod.Get, SuppressHttpError = true, Encoding = _encoding }; requestBuilder.Headers.Add("Referer", SiteLink); var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); Cookies = response.GetCookies(); CheckForError(response, login.Error); CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); } else if (login.Method == "oneurl") { var oneUrl = ApplyGoTemplateText(login.Inputs["oneurl"]); var loginUrl = ResolvePath(login.Path + oneUrl).ToString(); CookiesUpdater(null, null); var requestBuilder = new HttpRequestBuilder(loginUrl) { LogResponseContent = true, Method = HttpMethod.Get, SuppressHttpError = true, Encoding = _encoding }; requestBuilder.Headers.Add("Referer", SiteLink); var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition); Cookies = response.GetCookies(); CheckForError(response, login.Error); CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); } else { throw new NotImplementedException("Login method " + login.Method + " not implemented"); } }