Beispiel #1
0
        /// <summary>
        /// Get Torrents Page from Tracker by Query Provided
        /// </summary>
        /// <param name="request">URL created by Query Builder</param>
        /// <returns>Results from query</returns>
        private async Task <string> QueryTracker(string request)
        {
            // Cache mode not enabled or cached file didn't exist for our query
            Output("\nQuerying tracker for results....");

            // Build WebRequest for index
            var myIndexRequest = new WebRequest
            {
                Type     = RequestType.GET,
                Url      = request,
                Encoding = Encoding,
                Headers  = EmulatedBrowserHeaders
            };

            // Request our first page
            var results = await webclient.GetResultAsync(myIndexRequest);

            if (results.Status == HttpStatusCode.InternalServerError) // See issue #2110
            {
                throw new Exception("Internal Server Error (" + results.ContentString + "), probably you reached the API limits, please reduce the number of queries");
            }

            // Return results from tracker
            return(results.ContentString);
        }
Beispiel #2
0
 public override Task <WebResult> GetResultAsync(WebRequest request)
 {
     if (_requestCallbacks.ContainsKey(request.Url))
     {
         return(Task.Factory.StartNew(
                    () =>
         {
             var response = _requestCallbacks[request.Url];
             response.Request = request;
             return response;
         }));
     }
     throw new Exception($"You have to mock the URL {request.Url} with RegisterRequestCallback");
 }
Beispiel #3
0
        protected override async Task <WebClientByteResult> Run(WebRequest webRequest)
        {
            ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;

            var cookies = new CookieContainer();

            if (!string.IsNullOrWhiteSpace(webRequest.Cookies))
            {
                // don't include the path, Scheme is needed for mono compatibility
                var requestUri       = new Uri(webRequest.Url);
                var cookieUrl        = new Uri(requestUri.Scheme + "://" + requestUri.Host);
                var cookieDictionary = CookieUtil.CookieHeaderToDictionary(webRequest.Cookies);
                foreach (var kv in cookieDictionary)
                {
                    cookies.Add(cookieUrl, new Cookie(kv.Key, kv.Value));
                }
            }

            var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion();

            using (var clearanceHandlr = new ClearanceHandler(userAgent))
            {
                clearanceHandlr.MaxTries = 10;
                using (var clientHandlr = new HttpClientHandler
                {
                    CookieContainer = cookies,
                    AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                    UseCookies = true,
                    Proxy = webProxy,
                    UseProxy = (webProxy != null),
                    AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
                })
                {
                    // custom certificate validation handler (netcore version)
                    clientHandlr.ServerCertificateCustomValidationCallback = ValidateCertificate;

                    clearanceHandlr.InnerHandler = clientHandlr;
                    using (var client = new HttpClient(clearanceHandlr))
                    {
                        //if (webRequest.EmulateBrowser == true)
                        //    client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
                        //else
                        //    client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());

                        HttpResponseMessage response = null;
                        using (var request = new HttpRequestMessage())
                        {
                            request.Headers.ExpectContinue = false;
                            request.RequestUri             = new Uri(webRequest.Url);

                            if (webRequest.Headers != null)
                            {
                                foreach (var header in webRequest.Headers)
                                {
                                    if (header.Key != "Content-Type")
                                    {
                                        request.Headers.TryAddWithoutValidation(header.Key, header.Value);
                                    }
                                }
                            }

                            if (!string.IsNullOrEmpty(webRequest.Referer))
                            {
                                request.Headers.Referrer = new Uri(webRequest.Referer);
                            }

                            if (!string.IsNullOrEmpty(webRequest.RawBody))
                            {
                                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
                                if (type.HasValue)
                                {
                                    var str = new StringContent(webRequest.RawBody);
                                    str.Headers.Remove("Content-Type");
                                    str.Headers.Add("Content-Type", type.Value.Value);
                                    request.Content = str;
                                }
                                else
                                {
                                    request.Content = new StringContent(webRequest.RawBody);
                                }
                                request.Method = HttpMethod.Post;
                            }
                            else if (webRequest.Type == RequestType.POST)
                            {
                                if (webRequest.PostData != null)
                                {
                                    request.Content = FormUrlEncodedContentWithEncoding(webRequest.PostData, webRequest.Encoding);
                                }
                                request.Method = HttpMethod.Post;
                            }
                            else
                            {
                                request.Method = HttpMethod.Get;
                            }

                            using (response = await client.SendAsync(request))
                            {
                                var result = new WebClientByteResult
                                {
                                    Content = await response.Content.ReadAsByteArrayAsync()
                                };

                                foreach (var header in response.Headers)
                                {
                                    var value = header.Value;
                                    result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
                                }

                                // some cloudflare clients are using a refresh header
                                // Pull it out manually
                                if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
                                {
                                    var refreshHeaders = response.Headers.GetValues("Refresh");
                                    var redirval       = "";
                                    var redirtime      = 0;
                                    if (refreshHeaders != null)
                                    {
                                        foreach (var value in refreshHeaders)
                                        {
                                            var start = value.IndexOf("=");
                                            var end   = value.IndexOf(";");
                                            var len   = value.Length;
                                            if (start > -1)
                                            {
                                                redirval             = value.Substring(start + 1);
                                                result.RedirectingTo = redirval;
                                                // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                                                // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                                                // it shoudln't include service unavailable..only if we have this redirect header.
                                                response.StatusCode = System.Net.HttpStatusCode.Redirect;
                                                redirtime           = int.Parse(value.Substring(0, end));
                                                System.Threading.Thread.Sleep(redirtime * 1000);
                                            }
                                        }
                                    }
                                }
                                if (response.Headers.Location != null)
                                {
                                    result.RedirectingTo = response.Headers.Location.ToString();
                                }
                                // Mono won't add the baseurl to relative redirects.
                                // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
                                // See issue #1200
                                if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
                                {
                                    // URL decoding apparently is needed to, without it e.g. Demonoid download is broken
                                    // TODO: is it always needed (not just for relative redirects)?
                                    var newRedirectingTo = WebUtilityHelpers.UrlDecode(result.RedirectingTo, webRequest.Encoding);
                                    if (newRedirectingTo.StartsWith("file:////")) // Location without protocol but with host (only add scheme)
                                    {
                                        newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + ":");
                                    }
                                    else
                                    {
                                        newRedirectingTo = newRedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
                                    }
                                    logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
                                    result.RedirectingTo = newRedirectingTo;
                                }
                                result.Status = response.StatusCode;

                                // Compatiblity issue between the cookie format and httpclient
                                // Pull it out manually ignoring the expiry date then set it manually
                                // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
                                var responseCookies = new List <Tuple <string, string> >();

                                if (response.Headers.TryGetValues("set-cookie", out var cookieHeaders))
                                {
                                    foreach (var value in cookieHeaders)
                                    {
                                        var nameSplit = value.IndexOf('=');
                                        if (nameSplit > -1)
                                        {
                                            responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
                                        }
                                    }

                                    var cookieBuilder = new StringBuilder();
                                    foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                                    {
                                        cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                                    }
                                    result.Cookies = cookieBuilder.ToString().Trim();
                                }
                                ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
                                return(result);
                            }
                        }
                    }
                }
            }
        }
Beispiel #4
0
        public static void RegisterStringCall(WebRequest r, Func <WebRequest, WebClientStringResult> f)
        {
            var client = testContainer.Resolve <WebClient>() as TestWebClient;

            client.RegisterStringCall(r, f);
        }