/// <summary> /// Get Torrents Page from Tracker by Query Provided /// </summary> /// <param name="request">URL created by Query Builder</param> /// <returns>Results from query</returns> private async Task <string> QueryTracker(string request) { // Cache mode not enabled or cached file didn't exist for our query Output("\nQuerying tracker for results...."); // Build WebRequest for index var myIndexRequest = new WebRequest { Type = RequestType.GET, Url = request, Encoding = Encoding, Headers = EmulatedBrowserHeaders }; // Request our first page var results = await webclient.GetResultAsync(myIndexRequest); if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110 { throw new Exception("Internal Server Error (" + results.ContentString + "), probably you reached the API limits, please reduce the number of queries"); } // Return results from tracker return(results.ContentString); }
public override Task <WebResult> GetResultAsync(WebRequest request) { if (_requestCallbacks.ContainsKey(request.Url)) { return(Task.Factory.StartNew( () => { var response = _requestCallbacks[request.Url]; response.Request = request; return response; })); } throw new Exception($"You have to mock the URL {request.Url} with RegisterRequestCallback"); }
protected override async Task <WebClientByteResult> Run(WebRequest webRequest) { ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; var cookies = new CookieContainer(); if (!string.IsNullOrWhiteSpace(webRequest.Cookies)) { // don't include the path, Scheme is needed for mono compatibility var requestUri = new Uri(webRequest.Url); var cookieUrl = new Uri(requestUri.Scheme + "://" + requestUri.Host); var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies); foreach (var kv in cookieDictionary) { cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value)); } } var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion(); using (var clearanceHandlr = new ClearanceHandler(userAgent)) { clearanceHandlr.MaxTries = 10; using (var clientHandlr = new HttpClientHandler { CookieContainer = cookies, AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. UseCookies = true, Proxy = webProxy, UseProxy = (webProxy != null), AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }) { // custom certificate validation handler (netcore version) clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate; clearanceHandlr.InnerHandler = clientHandlr; using (var client = new HttpClient(clearanceHandlr)) { //if (webRequest.EmulateBrowser == true) // client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); //else // client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion()); HttpResponseMessage response = null; using (var request = new HttpRequestMessage()) { request.Headers.ExpectContinue = false; request.RequestUri = new Uri(webRequest.Url); if (webRequest.Headers != null) { foreach (var header in webRequest.Headers) { if (header.Key != "Content-Type") { request.Headers.TryAddWithoutValidation(header.Key, header.Value); } } } if (!string.IsNullOrEmpty(webRequest.Referer)) { request.Headers.Referrer = new Uri(webRequest.Referer); } if (!string.IsNullOrEmpty(webRequest.RawBody)) { var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault(); if (type.HasValue) { var str = new StringContent(webRequest.RawBody); str.Headers.Remove("Content-Type"); str.Headers.Add("Content-Type", type.Value.Value); request.Content = str; } else { request.Content = new StringContent(webRequest.RawBody); } request.Method = HttpMethod.Post; } else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) { request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding); } request.Method = HttpMethod.Post; } else { request.Method = HttpMethod.Get; } using (response = await client.SendAsync(request)) { var result = new WebClientByteResult { Content = await response.Content.ReadAsByteArrayAsync() }; foreach (var header in response.Headers) { var value = header.Value; result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); } // some cloudflare clients are using a refresh header // Pull it out manually if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) { var refreshHeaders = response.Headers.GetValues("Refresh"); var redirval = ""; var redirtime = 0; if (refreshHeaders != null) { foreach (var value in refreshHeaders) { var start = value.IndexOf("="); var end = value.IndexOf(";"); var len = value.Length; if (start > -1) { redirval = value.Substring(start + 1); result.RedirectingTo = redirval; // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally // it shoudln't include service unavailable..only if we have this redirect header. response.StatusCode = System.Net.HttpStatusCode.Redirect; redirtime = int.Parse(value.Substring(0, end)); System.Threading.Thread.Sleep(redirtime * 1000); } } } } if (response.Headers.Location != null) { result.RedirectingTo = response.Headers.Location.ToString(); } // Mono won't add the baseurl to relative redirects. // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" // See issue #1200 if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) { // URL decoding apparently is needed to, without it e.g. Demonoid download is broken // TODO: is it always needed (not just for relative redirects)? var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding); if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme) { newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":"); } else { newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); } logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); result.RedirectingTo = newRedirectingTo; } result.Status = response.StatusCode; // Compatiblity issue between the cookie format and httpclient // Pull it out manually ignoring the expiry date then set it manually // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer var responseCookies = new List <Tuple <string, string> >(); if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders)) { foreach (var value in cookieHeaders) { var nameSplit = value.IndexOf('='); if (nameSplit > -1) { responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); } } var cookieBuilder = new StringBuilder(); foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) { cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); } result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); return(result); } } } } } }
public static void RegisterStringCall(WebRequest r, Func <WebRequest, WebClientStringResult> f) { var client = testContainer.Resolve <WebClient>() as TestWebClient; client.RegisterStringCall(r, f); }