Пример #1
0
        // Wrapper for Run which takes care of CloudFlare challenges
        private async Task <WebClientByteResult> RunCloudFlare(WebRequest request)
        {
            WebClientByteResult result = await Run(request);

            // check if we've received a CloudFlare challenge
            if (result.Status == HttpStatusCode.ServiceUnavailable && ((request.Cookies != null && request.Cookies.Contains("__cfduid")) || result.Cookies.Contains("__cfduid")))
            {
                logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge");

                // solve the challenge
                string pageContent  = Encoding.UTF8.GetString(result.Content);
                Uri    uri          = new Uri(request.Url);
                string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri);
                logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri));

                // wait...
                await Task.Delay(5000);

                // request clearanceUri to get cf_clearance cookie
                var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer);

                logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies));

                // add new cf_clearance cookies to the original request
                request.Cookies = response.Cookies + request.Cookies;

                // re-run the original request with updated cf_clearance cookie
                result = await Run(request);

                // add cf_clearance cookie to the final result so we update the config for the next request
                result.Cookies = response.Cookies + " " + result.Cookies;
            }
            return(result);
        }
Пример #2
0
        public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified()
        {
            var res = new WebClientByteResult()
            {
                RedirectingTo = "list?p=1"
            };

            var req = new WebRequest()
            {
                Url = "http://my.domain.com/page.php"
            };

            // Not fully qualified  requiring redirect
            ServerUtil.ResureRedirectIsFullyQualified(req, res);
            Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1");

            // Fully qualified not needing modified
            res.RedirectingTo = "http://a.domain/page.htm";
            ServerUtil.ResureRedirectIsFullyQualified(req, res);
            Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm");

            // Relative  requiring redirect
            req.Url = "http://my.domain.com/dir/page.php";
            res.RedirectingTo = "a/dir/page.html";
            ServerUtil.ResureRedirectIsFullyQualified(req, res);
            Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html");
        }
Пример #3
0
        // Wrapper for Run which takes care of CloudFlare challenges, calls RunCurl
        override protected async Task <WebClientByteResult> Run(WebRequest request)
        {
            WebClientByteResult result = await RunCurl(request);

            // check if we've received a CloudFlare challenge
            string[] server;
            if (result.Status == HttpStatusCode.ServiceUnavailable && result.Headers.TryGetValue("server", out server) && server[0] == "cloudflare-nginx")
            {
                logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge");

                // solve the challenge
                string pageContent  = Encoding.UTF8.GetString(result.Content);
                Uri    uri          = new Uri(request.Url);
                string clearanceUri = CloudFlareChallengeSolverSolve(pageContent, uri);
                logger.Info(string.Format("UnixLibCurlWebClient: CloudFlare clearanceUri: {0}", clearanceUri));

                // wait...
                await Task.Delay(5000);

                // request clearanceUri to get cf_clearance cookie
                var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer);

                logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies));

                // add new cf_clearance cookies to the original request
                request.Cookies = response.Cookies + request.Cookies;

                // re-run the original request with updated cf_clearance cookie
                result = await RunCurl(request);

                // add cf_clearance cookie to the final result so we update the config for the next request
                result.Cookies = response.Cookies + " " + result.Cookies;
            }
            return(result);
        }
Пример #4
0
        private async Task <WebClientByteResult> Run(WebRequest request)
        {
            Jackett.CurlHelper.CurlResponse response;
            if (request.Type == RequestType.GET)
            {
                response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
            }
            else
            {
                if (!string.IsNullOrEmpty(request.RawBody))
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
                }
                else if (request.PostData != null && request.PostData.Count() > 0)
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
                }

                response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
            }

            var result = new WebClientByteResult()
            {
                Content = response.Content,
                Cookies = response.Cookies,
                Status  = response.Status
            };

            if (response.HeaderList != null)
            {
                foreach (var header in response.HeaderList)
                {
                    switch (header[0].ToLowerInvariant())
                    {
                    case "location":
                        result.RedirectingTo = header[1];
                        break;
                    }
                }
            }

            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return(result);
        }
Пример #5
0
        public async Task<WebClientByteResult> GetBytes(WebRequest request)
        {
            Jackett.CurlHelper.CurlResponse response;

            logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url));

            if (request.Type == RequestType.GET)
            {
                response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
            }
            else
            {
                response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
            }

            var result = new WebClientByteResult()
            {
                Content = response.Content,
                Cookies = response.CookieHeader,
                Status = response.Status
            };

            if (response.Headers != null)
            {
                foreach(var header in response.Headers)
                {
                    if(string.Equals(header.Key, "location", StringComparison.InvariantCultureIgnoreCase) && header.Value !=null)
                    {
                        result.RedirectingTo = header.Value;
                    }
                }
            }

            logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status));
            return result;
        }
Пример #6
0
        private async Task<WebClientByteResult> Run(WebRequest request)
        {
            Jackett.CurlHelper.CurlResponse response;
            if (request.Type == RequestType.GET)
            {
                response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
            }
            else
            {
                if (!string.IsNullOrEmpty(request.RawBody))
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
                }
                else if (request.PostData != null && request.PostData.Count() > 0)
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
                }

                response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
            }

            var result = new WebClientByteResult()
            {
                Content = response.Content,
                Cookies = response.Cookies,
                Status = response.Status
            };

            if (response.HeaderList != null)
            {
                foreach (var header in response.HeaderList)
                {
                    switch (header[0].ToLowerInvariant())
                    {
                        case "location":
                            result.RedirectingTo = header[1];
                            break;
                        case "refresh":
                            if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable)
                            {
                                //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
                                var redirval = "";
                                var value = header[1];
                                var start = value.IndexOf("=");
                                var end = value.IndexOf(";");
                                var len = value.Length;
                                if (start > -1)
                                {
                                    redirval = value.Substring(start + 1);
                                    result.RedirectingTo = redirval;
                                    // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                                    // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                                    // it shoudln't include service unavailable..only if we have this redirect header.
                                    result.Status = System.Net.HttpStatusCode.Redirect;
                                    var redirtime = Int32.Parse(value.Substring(0, end));
                                    System.Threading.Thread.Sleep(redirtime * 1000);
                                }
                            }
                            break;
                    }
                }
            }

            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return result;
        }
Пример #7
0
        private async Task<WebClientByteResult> Run(WebRequest request)
        {
            var args = new StringBuilder();
            args.AppendFormat("--url \"{0}\" ", request.Url);
            args.AppendFormat("-i  -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);

            if (!string.IsNullOrWhiteSpace(request.Cookies))
            {
                args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
            }

            if (!string.IsNullOrWhiteSpace(request.Referer))
            {
                args.AppendFormat("--referer \"{0}\" ", request.Referer);
            }

            if (request.PostData != null && request.PostData.Count > 0)
            {
                var postString = new FormUrlEncodedContent(request.PostData).ReadAsStringAsync().Result;
                args.AppendFormat("--data \"{0}\" ", postString);
            }

            var tempFile = Path.GetTempFileName();
            args.AppendFormat("--output \"{0}\" ", tempFile);

            if (Startup.DoSSLFix == true)
            {
                // http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
                // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
                args.Append("--cipher " + SSLFix.CipherList);
            }

            string stdout = null;
            await Task.Run(() =>
            {
                stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix?"curl":"curl.exe", args.ToString(), true);
            });

            var outputData = File.ReadAllBytes(tempFile);
            File.Delete(tempFile);
            stdout = Encoding.UTF8.GetString(outputData);
            var result = new WebClientByteResult();
            var headSplit = stdout.IndexOf("\r\n\r\n");
            if (headSplit < 0)
                throw new Exception("Invalid response");
            var headers = stdout.Substring(0, headSplit);
            var headerCount = 0;
            foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
            {
                if (headerCount == 0)
                {
                    var responseCode = int.Parse(header.Split(' ')[1]);
                    result.Status = (HttpStatusCode)responseCode;
                }
                else
                {
                    var headerSplitIndex = header.IndexOf(':');
                    if (headerSplitIndex > 0)
                    {
                        var name = header.Substring(0, headerSplitIndex).ToLowerInvariant();
                        var value = header.Substring(headerSplitIndex + 1);
                        switch (name)
                        {
                            case "set-cookie":
                                var cookieDataSplit = value.IndexOf(';');
                                if (cookieDataSplit > 0)
                                {
                                    result.Cookies += value.Substring(0, cookieDataSplit + 1) + " ";
                                }//Location
                                break;
                            case "location":
                                result.RedirectingTo = value.Trim();
                                break;
                        }
                    }
                }
                headerCount++;
            }

            result.Content = new byte[outputData.Length - (headSplit + 3)];
            var dest = 0;
            for (int i= headSplit+4;i< outputData.Length; i++)
            {
                result.Content[dest] = outputData[i];
                dest++;
            }

            logger.Debug("WebClientByteResult returned " + result.Status);
            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return result;
        }
Пример #8
0
        private async Task<WebClientByteResult> Run(WebRequest request)
        {
            var cookies = new CookieContainer();
            if (!string.IsNullOrEmpty(request.Cookies))
            {
                var uri = new Uri(request.Url);
                foreach (var c in request.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(uri, c);
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }

            var client = new HttpClient(new HttpClientHandler
            {
                CookieContainer = cookies,
                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                UseCookies = true,
            });

            client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
            HttpResponseMessage response = null;
           
            if (request.Type == RequestType.POST)
            {
                var content = new FormUrlEncodedContent(request.PostData);
                response = await client.PostAsync(request.Url, content);
            }
            else
            {
                response = await client.GetAsync(request.Url);
            }

            var result = new WebClientByteResult();
            result.Content = await response.Content.ReadAsByteArrayAsync();
            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable<string> cookieHeaders;
            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                var cookieBuilder = new StringBuilder();
                foreach (var c in cookieHeaders)
                {
                    cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';')+1));
                }

                result.Cookies = cookieBuilder.ToString().TrimEnd();
            }

            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return result;
        }
Пример #9
0
        override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
        {
            HttpResponseMessage response = null;
            var request = new HttpRequestMessage();

            request.Headers.ExpectContinue = false;
            request.RequestUri             = new Uri(webRequest.Url);

            if (webRequest.EmulateBrowser == true)
            {
                request.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent);
            }
            else
            {
                request.Headers.UserAgent.ParseAdd("Jackett/" + configService.GetVersion());
            }

            // clear cookies from cookiecontainer
            var oldCookies = cookies.GetCookies(request.RequestUri);

            foreach (Cookie oldCookie in oldCookies)
            {
                oldCookie.Expired = true;
            }

            if (!string.IsNullOrEmpty(webRequest.Cookies))
            {
                // add cookies to cookiecontainer
                var cookieUrl = new Uri(request.RequestUri.Scheme + "://" + request.RequestUri.Host); // don't include the path, Scheme is needed for mono compatibility
                foreach (var ccookiestr in webRequest.Cookies.Split(';'))
                {
                    var cookiestrparts = ccookiestr.Split('=');
                    var name           = cookiestrparts[0].Trim();
                    if (string.IsNullOrWhiteSpace(name))
                    {
                        continue;
                    }
                    var value = "";
                    if (cookiestrparts.Length >= 2)
                    {
                        value = cookiestrparts[1].Trim();
                    }
                    var cookie = new Cookie(name, value);
                    cookies.Add(cookieUrl, cookie);
                }
            }

            if (webRequest.Headers != null)
            {
                foreach (var header in webRequest.Headers)
                {
                    if (header.Key != "Content-Type")
                    {
                        request.Headers.TryAddWithoutValidation(header.Key, header.Value);
                    }
                }
            }

            if (!string.IsNullOrEmpty(webRequest.Referer))
            {
                request.Headers.Referrer = new Uri(webRequest.Referer);
            }

            if (!string.IsNullOrEmpty(webRequest.RawBody))
            {
                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
                if (type.HasValue)
                {
                    var str = new StringContent(webRequest.RawBody);
                    str.Headers.Remove("Content-Type");
                    str.Headers.Add("Content-Type", type.Value.Value);
                    request.Content = str;
                }
                else
                {
                    request.Content = new StringContent(webRequest.RawBody);
                }
                request.Method = HttpMethod.Post;
            }
            else if (webRequest.Type == RequestType.POST)
            {
                if (webRequest.PostData != null)
                {
                    request.Content = new FormUrlEncodedContent(webRequest.PostData);
                }
                request.Method = HttpMethod.Post;
            }
            else
            {
                request.Method = HttpMethod.Get;
            }

            response = await client.SendAsync(request);

            var result = new WebClientByteResult();

            result.Content = await response.Content.ReadAsByteArrayAsync();

            foreach (var header in response.Headers)
            {
                IEnumerable <string> value = header.Value;
                result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
            }

            // some cloudflare clients are using a refresh header
            // Pull it out manually
            if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
            {
                var refreshHeaders = response.Headers.GetValues("Refresh");
                var redirval       = "";
                var redirtime      = 0;
                if (refreshHeaders != null)
                {
                    foreach (var value in refreshHeaders)
                    {
                        var start = value.IndexOf("=");
                        var end   = value.IndexOf(";");
                        var len   = value.Length;
                        if (start > -1)
                        {
                            redirval             = value.Substring(start + 1);
                            result.RedirectingTo = redirval;
                            // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                            // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                            // it shoudln't include service unavailable..only if we have this redirect header.
                            response.StatusCode = System.Net.HttpStatusCode.Redirect;
                            redirtime           = Int32.Parse(value.Substring(0, end));
                            System.Threading.Thread.Sleep(redirtime * 1000);
                        }
                    }
                }
            }
            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            // Mono won't add the baseurl to relative redirects.
            // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
            // See issue #1200
            if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
            {
                var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
                logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
                result.RedirectingTo = newRedirectingTo;
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable <string> cookieHeaders;
            var responseCookies = new List <Tuple <string, string> >();

            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                foreach (var value in cookieHeaders)
                {
                    logger.Debug(value);
                    var nameSplit = value.IndexOf('=');
                    if (nameSplit > -1)
                    {
                        responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
                    }
                }

                var cookieBuilder = new StringBuilder();
                foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                {
                    cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                }
                result.Cookies = cookieBuilder.ToString().Trim();
            }
            ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
            return(result);
        }
Пример #10
0
        private async Task<WebClientByteResult> Run(WebRequest webRequest)
        {
            var cookies = new CookieContainer();
            if (!string.IsNullOrEmpty(webRequest.Cookies))
            {
                var uri = new Uri(webRequest.Url);
                foreach (var c in webRequest.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(uri, c);
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }

            var client = new HttpClient(new HttpClientHandler
            {
                CookieContainer = cookies,
                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                UseCookies = true,
            });

            if(webRequest.EmulateBrowser)
               client.DefaultRequestHeaders.Add("User-Agent",  BrowserUtil.ChromeUserAgent);
            else
               client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
            HttpResponseMessage response = null;
            var request = new HttpRequestMessage();
            request.Headers.ExpectContinue = false;
            request.RequestUri = new Uri(webRequest.Url);

            if (webRequest.Headers != null)
            {
                foreach (var header in webRequest.Headers)
                {
                    if (header.Key != "Content-Type")
                    {
                        request.Headers.Add(header.Key, header.Value);
                    }
                }
            }

            if (!string.IsNullOrEmpty(webRequest.RawBody))
            {
                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
                if (type.HasValue)
                {
                    var str = new StringContent(webRequest.RawBody);
                    str.Headers.Remove("Content-Type");
                    str.Headers.Add("Content-Type", type.Value.Value);
                    request.Content = str;
                }
                else
                    request.Content = new StringContent(webRequest.RawBody);
                request.Method = HttpMethod.Post;
            }
            else if (webRequest.Type == RequestType.POST)
            {
                request.Content = new FormUrlEncodedContent(webRequest.PostData);
                request.Method = HttpMethod.Post;
            }
            else
            {
                request.Method = HttpMethod.Get;
            }

            response = await client.SendAsync(request);

            var result = new WebClientByteResult();
            result.Content = await response.Content.ReadAsByteArrayAsync();
            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable<string> cookieHeaders;
            var responseCookies = new List<Tuple<string, string>>();
            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                foreach (var value in cookieHeaders)
                {
                    var nameSplit = value.IndexOf('=');
                    if (nameSplit > -1)
                    {
                        responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
                    }
                }

                var cookieBuilder = new StringBuilder();
                foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                {
                    cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                }
                result.Cookies = cookieBuilder.ToString().Trim();
            }

            ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
            return result;
        }
Пример #11
0
        private async Task<WebClientByteResult> Run(WebRequest request)
        {
            Jackett.CurlHelper.CurlResponse response;
            if (request.Type == RequestType.GET)
            {
                response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
            }
            else
            {
                if (request.PostData != null && request.PostData.Count() > 0)
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
                }

                response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
            }

            var result = new WebClientByteResult()
            {
                Content = response.Content,
                Cookies = response.Cookies,
                Status = response.Status
            };

            if (response.HeaderList != null)
            {
                foreach (var header in response.HeaderList)
                {
                    switch (header[0].ToLowerInvariant())
                    {
                        case "location":
                            result.RedirectingTo = header[1];
                            break;
                    }
                }
            }

            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return result;
        }
Пример #12
0
        private async Task <WebClientByteResult> Run(WebRequest request)
        {
            var cookies = new CookieContainer();

            if (!string.IsNullOrEmpty(request.Cookies))
            {
                var uri = new Uri(request.Url);
                foreach (var c in request.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(uri, c);
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }

            var client = new HttpClient(new HttpClientHandler
            {
                CookieContainer   = cookies,
                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                UseCookies        = true,
            });

            client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
            HttpResponseMessage response = null;

            if (request.Type == RequestType.POST)
            {
                var content = new FormUrlEncodedContent(request.PostData);
                response = await client.PostAsync(request.Url, content);
            }
            else
            {
                response = await client.GetAsync(request.Url);
            }

            var result = new WebClientByteResult();

            result.Content = await response.Content.ReadAsByteArrayAsync();

            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable <string> cookieHeaders;

            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                var cookieBuilder = new StringBuilder();
                foreach (var c in cookieHeaders)
                {
                    cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1));
                }

                result.Cookies = cookieBuilder.ToString().TrimEnd();
            }

            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return(result);
        }
Пример #13
0
 protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
 {
     // Follow up  to 5 redirects
     for (int i = 0; i < 5; i++)
     {
         if (!response.IsRedirect)
             break;
         await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies);
     }
 }
Пример #14
0
        override protected async Task <WebClientByteResult> Run(WebRequest request)
        {
            var args = new StringBuilder();

            if (Startup.ProxyConnection != null)
            {
                args.AppendFormat("-x " + Startup.ProxyConnection + " ");
            }

            args.AppendFormat("--url \"{0}\" ", request.Url);

            if (request.EmulateBrowser == true)
            {
                args.AppendFormat("-i  -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
            }
            else
            {
                args.AppendFormat("-i  -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());
            }

            if (!string.IsNullOrWhiteSpace(request.Cookies))
            {
                args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
            }

            if (!string.IsNullOrWhiteSpace(request.Referer))
            {
                args.AppendFormat("--referer \"{0}\" ", request.Referer);
            }

            if (!string.IsNullOrEmpty(request.RawBody))
            {
                var postString = StringUtil.PostDataFromDict(request.PostData);
                args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
            }
            else if (request.PostData != null && request.PostData.Count() > 0)
            {
                var postString = StringUtil.PostDataFromDict(request.PostData);
                args.AppendFormat("--data \"{0}\" ", postString);
            }

            var tempFile = Path.GetTempFileName();

            args.AppendFormat("--output \"{0}\" ", tempFile);

            if (Startup.DoSSLFix == true)
            {
                // http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
                // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
                args.Append("--cipher " + SSLFix.CipherList);
            }
            if (Startup.IgnoreSslErrors == true)
            {
                args.Append("-k ");
            }
            args.Append("-H \"Accept-Language: en-US,en\" ");
            args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" ");
            string stdout = null;
            await Task.Run(() =>
            {
                stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
            });

            var outputData = File.ReadAllBytes(tempFile);

            File.Delete(tempFile);
            stdout = Encoding.UTF8.GetString(outputData);
            var result    = new WebClientByteResult();
            var headSplit = stdout.IndexOf("\r\n\r\n");

            if (headSplit < 0)
            {
                throw new Exception("Invalid response");
            }
            var headers = stdout.Substring(0, headSplit);

            if (Startup.ProxyConnection != null)
            {
                // the proxy provided headers too so we need to split headers again
                var headSplit1 = stdout.IndexOf("\r\n\r\n", headSplit + 4);
                if (headSplit1 > 0)
                {
                    headers   = stdout.Substring(headSplit + 4, headSplit1 - (headSplit + 4));
                    headSplit = headSplit1;
                }
            }
            var headerCount   = 0;
            var cookieBuilder = new StringBuilder();
            var cookies       = new List <Tuple <string, string> >();

            foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
            {
                if (headerCount == 0)
                {
                    var responseCode = int.Parse(header.Split(' ')[1]);
                    result.Status = (HttpStatusCode)responseCode;
                }
                else
                {
                    var headerSplitIndex = header.IndexOf(':');
                    if (headerSplitIndex > 0)
                    {
                        var name  = header.Substring(0, headerSplitIndex).ToLowerInvariant();
                        var value = header.Substring(headerSplitIndex + 1);
                        switch (name)
                        {
                        case "set-cookie":
                            var nameSplit = value.IndexOf('=');
                            if (nameSplit > -1)
                            {
                                cookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
                            }
                            break;

                        case "location":
                            result.RedirectingTo = value.Trim();
                            break;

                        case "refresh":
                            //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
                            var redirval = "";
                            var start    = value.IndexOf("=");
                            var end      = value.IndexOf(";");
                            var len      = value.Length;
                            if (start > -1)
                            {
                                redirval             = value.Substring(start + 1);
                                result.RedirectingTo = redirval;
                                // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                                // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                                // it shoudln't include service unavailable..only if we have this redirect header.
                                result.Status = System.Net.HttpStatusCode.Redirect;
                                var redirtime = Int32.Parse(value.Substring(0, end));
                                System.Threading.Thread.Sleep(redirtime * 1000);
                            }
                            break;
                        }
                    }
                }
                headerCount++;
            }

            foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
            {
                cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
            }

            result.Cookies = cookieBuilder.ToString().Trim();
            result.Content = new byte[outputData.Length - (headSplit + 3)];
            var dest = 0;

            for (int i = headSplit + 4; i < outputData.Length; i++)
            {
                result.Content[dest] = outputData[i];
                dest++;
            }

            logger.Debug("WebClientByteResult returned " + result.Status);
            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return(result);
        }
Пример #15
0
        protected async Task <WebClientByteResult> RunCurl(WebRequest request)
        {
            Jackett.CurlHelper.CurlResponse response;
            if (request.Type == RequestType.GET)
            {
                response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer, request.Headers);
            }
            else
            {
                if (!string.IsNullOrEmpty(request.RawBody))
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
                }
                else if (request.PostData != null && request.PostData.Count() > 0)
                {
                    logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
                }

                response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.Headers, request.RawBody);
            }

            var result = new WebClientByteResult()
            {
                Content = response.Content,
                Cookies = response.Cookies,
                Status  = response.Status
            };

            if (response.HeaderList != null)
            {
                foreach (var header in response.HeaderList)
                {
                    var key = header[0].ToLowerInvariant();

                    result.Headers[key] = new string[] { header[1] }; // doesn't support multiple identical headers?

                    switch (key)
                    {
                    case "location":
                        result.RedirectingTo = header[1];
                        break;

                    case "refresh":
                        if (response.Status == System.Net.HttpStatusCode.ServiceUnavailable)
                        {
                            //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
                            var redirval = "";
                            var value    = header[1];
                            var start    = value.IndexOf("=");
                            var end      = value.IndexOf(";");
                            var len      = value.Length;
                            if (start > -1)
                            {
                                redirval             = value.Substring(start + 1);
                                result.RedirectingTo = redirval;
                                // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                                // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                                // it shoudln't include service unavailable..only if we have this redirect header.
                                result.Status = System.Net.HttpStatusCode.Redirect;
                                var redirtime = Int32.Parse(value.Substring(0, end));
                                System.Threading.Thread.Sleep(redirtime * 1000);
                            }
                        }
                        break;
                    }
                }
            }

            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return(result);
        }
Пример #16
0
 private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
 {
     if (incomingResponse.IsRedirect)
     {
         var redirRequestCookies = "";
         if (accumulateCookies)
         {
             redirRequestCookies = ResolveCookies((CookieHeader != "" ? CookieHeader + " " : "") + (overrideCookies != null ? overrideCookies : ""));
         } else
         {
             redirRequestCookies = (overrideCookies != null ? overrideCookies : "");
         }
         // Do redirect
         var redirectedResponse = await webclient.GetBytes(new WebRequest()
         {
             Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
             Referer = referrer,
             Cookies = redirRequestCookies
         });
         Mapper.Map(redirectedResponse, incomingResponse);
     }
 }
Пример #17
0
 protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null, bool accumulateCookies = false)
 {
     // Follow up  to 5 redirects
     for (int i = 0; i < 5; i++)
     {
         if (!response.IsRedirect)
             break;
         await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies, accumulateCookies);
         if (accumulateCookies)
         {
             CookieHeader = ResolveCookies((CookieHeader != null && CookieHeader != ""? CookieHeader + " " : "") + (overrideCookies != null && overrideCookies != "" ? overrideCookies + " " : "") + response.Cookies);
             overrideCookies = response.Cookies = CookieHeader;
         }
         if (overrideCookies != null && response.Cookies == null)
         {
             response.Cookies = overrideCookies;
         }
     }
 }
Пример #18
0
        private async Task<WebClientByteResult> Run(WebRequest request)
        {
            var args = new StringBuilder();
            if (Startup.ProxyConnection != null)
            {
                args.AppendFormat("-x " + Startup.ProxyConnection + " ");
            }
            
            args.AppendFormat("--url \"{0}\" ", request.Url);
           
            if (request.EmulateBrowser)
                args.AppendFormat("-i  -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
            else
                args.AppendFormat("-i  -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());

            if (!string.IsNullOrWhiteSpace(request.Cookies))
            {
                args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
            }

            if (!string.IsNullOrWhiteSpace(request.Referer))
            {
                args.AppendFormat("--referer \"{0}\" ", request.Referer);
            }

            if (!string.IsNullOrEmpty(request.RawBody))
            {
                var postString = StringUtil.PostDataFromDict(request.PostData);
                args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
            } else if (request.PostData != null && request.PostData.Count() > 0)
            {
                var postString = StringUtil.PostDataFromDict(request.PostData);
                args.AppendFormat("--data \"{0}\" ", postString);
            }

            var tempFile = Path.GetTempFileName();
            args.AppendFormat("--output \"{0}\" ", tempFile);

            if (Startup.DoSSLFix == true)
            {
                // http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
                // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
                args.Append("--cipher " + SSLFix.CipherList);
            }
            if (Startup.IgnoreSslErrors == true)
            {
                args.Append("-k ");
            }
            args.Append("-H \"Accept-Language: en-US,en\" ");
            args.Append("-H \"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\" ");
            string stdout = null;
            await Task.Run(() =>
            {
                stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString() , true);
            });

            var outputData = File.ReadAllBytes(tempFile);
            File.Delete(tempFile);
            stdout = Encoding.UTF8.GetString(outputData);
            var result = new WebClientByteResult();
            var headSplit = stdout.IndexOf("\r\n\r\n");
            if (headSplit < 0)
                throw new Exception("Invalid response");
            var headers = stdout.Substring(0, headSplit);
            if (Startup.ProxyConnection != null)
            {
                // the proxy provided headers too so we need to split headers again
                var headSplit1 = stdout.IndexOf("\r\n\r\n",headSplit + 4);
                if (headSplit1 > 0)
                {
                    headers = stdout.Substring(headSplit + 4,headSplit1 - (headSplit + 4));
                    headSplit = headSplit1;
                }
            }
            var headerCount = 0;
            var cookieBuilder = new StringBuilder();
            var cookies = new List<Tuple<string, string>>();

            foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
            {
                if (headerCount == 0)
                {
                    var responseCode = int.Parse(header.Split(' ')[1]);
                    result.Status = (HttpStatusCode)responseCode;
                }
                else
                {
                    var headerSplitIndex = header.IndexOf(':');
                    if (headerSplitIndex > 0)
                    {
                        var name = header.Substring(0, headerSplitIndex).ToLowerInvariant();
                        var value = header.Substring(headerSplitIndex + 1);
                        switch (name)
                        {
                            case "set-cookie":
                                var nameSplit = value.IndexOf('=');
                                if (nameSplit > -1)
                                {
                                    cookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
                                }
                                break;
                            case "location":
                                result.RedirectingTo = value.Trim();
                                break;
                            case "refresh":
                                //"Refresh: 8;URL=/cdn-cgi/l/chk_jschl?pass=1451000679.092-1vJFUJLb9R"
                                var redirval = "";
                                var start = value.IndexOf("=");
                                var end = value.IndexOf(";");
                                var len = value.Length;
                                if (start > -1)
                                {
                                    redirval = value.Substring(start + 1);
                                    result.RedirectingTo = redirval;
                                    // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                                    // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                                    // it shoudln't include service unavailable..only if we have this redirect header.
                                    result.Status = System.Net.HttpStatusCode.Redirect;
                                    var redirtime = Int32.Parse(value.Substring(0, end));
                                    System.Threading.Thread.Sleep(redirtime * 1000);
                                }
                                break;
                        }
                    }
                }
                headerCount++;
            }

            foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
            {
                cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
            }

            result.Cookies = cookieBuilder.ToString().Trim();
            result.Content = new byte[outputData.Length - (headSplit + 3)];
            var dest = 0;
            for (int i = headSplit + 4; i < outputData.Length; i++)
            {
                result.Content[dest] = outputData[i];
                dest++;
            }

            logger.Debug("WebClientByteResult returned " + result.Status);
            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return result;
        }
Пример #19
0
        private async Task <WebClientByteResult> Run(WebRequest webRequest)
        {
            var cookies = new CookieContainer();

            if (!string.IsNullOrEmpty(webRequest.Cookies))
            {
                var uri = new Uri(webRequest.Url);
                foreach (var c in webRequest.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(uri, c);
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }

            var client = new HttpClient(new HttpClientHandler
            {
                CookieContainer   = cookies,
                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                UseCookies        = true,
            });

            if (webRequest.EmulateBrowser)
            {
                client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
            }
            else
            {
                client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
            }
            HttpResponseMessage response = null;
            var request = new HttpRequestMessage();

            request.Headers.ExpectContinue = false;
            request.RequestUri             = new Uri(webRequest.Url);

            if (webRequest.Headers != null)
            {
                foreach (var header in webRequest.Headers)
                {
                    if (header.Key != "Content-Type")
                    {
                        request.Headers.Add(header.Key, header.Value);
                    }
                }
            }

            if (!string.IsNullOrEmpty(webRequest.RawBody))
            {
                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
                if (type.HasValue)
                {
                    var str = new StringContent(webRequest.RawBody);
                    str.Headers.Remove("Content-Type");
                    str.Headers.Add("Content-Type", type.Value.Value);
                    request.Content = str;
                }
                else
                {
                    request.Content = new StringContent(webRequest.RawBody);
                }
                request.Method = HttpMethod.Post;
            }
            else if (webRequest.Type == RequestType.POST)
            {
                request.Content = new FormUrlEncodedContent(webRequest.PostData);
                request.Method  = HttpMethod.Post;
            }
            else
            {
                request.Method = HttpMethod.Get;
            }

            response = await client.SendAsync(request);

            var result = new WebClientByteResult();

            result.Content = await response.Content.ReadAsByteArrayAsync();

            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable <string> cookieHeaders;
            var responseCookies = new List <Tuple <string, string> >();

            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                foreach (var value in cookieHeaders)
                {
                    var nameSplit = value.IndexOf('=');
                    if (nameSplit > -1)
                    {
                        responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
                    }
                }

                var cookieBuilder = new StringBuilder();
                foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                {
                    cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                }
                result.Cookies = cookieBuilder.ToString().Trim();
            }

            ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
            return(result);
        }
Пример #20
0
        private async Task<WebClientByteResult> Run(WebRequest webRequest)
        {
            ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;

            var cookies = new CookieContainer();
            if (!string.IsNullOrEmpty(webRequest.Cookies))
            {
                var uri = new Uri(webRequest.Url);
                foreach (var c in webRequest.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(uri, c);
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }
            var useProxy = false;
            WebProxy proxyServer = null;
            if (Startup.ProxyConnection != null)
            {
                proxyServer = new WebProxy(Startup.ProxyConnection, false);
                useProxy = true;
            }
            var client = new HttpClient(new HttpClientHandler
            {
                CookieContainer = cookies,
                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                UseCookies = true,
                Proxy = proxyServer,
                UseProxy = useProxy
            });
            

            if (webRequest.EmulateBrowser)
                client.DefaultRequestHeaders.Add("User-Agent",  BrowserUtil.ChromeUserAgent);
            else
                client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
            
            HttpResponseMessage response = null;
            var request = new HttpRequestMessage();
            request.Headers.ExpectContinue = false;
            request.RequestUri = new Uri(webRequest.Url);

            if (webRequest.Headers != null)
            {
                foreach (var header in webRequest.Headers)
                {
                    if (header.Key != "Content-Type")
                    {
                        request.Headers.Add(header.Key, header.Value);
                    }
                }
            }

            if (!string.IsNullOrEmpty(webRequest.RawBody))
            {
                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
                if (type.HasValue)
                {
                    var str = new StringContent(webRequest.RawBody);
                    str.Headers.Remove("Content-Type");
                    str.Headers.Add("Content-Type", type.Value.Value);
                    request.Content = str;
                }
                else
                    request.Content = new StringContent(webRequest.RawBody);
                request.Method = HttpMethod.Post;
            }
            else if (webRequest.Type == RequestType.POST)
            {
                request.Content = new FormUrlEncodedContent(webRequest.PostData);
                request.Method = HttpMethod.Post;
            }
            else
            {
                request.Method = HttpMethod.Get;
            }

            response = await client.SendAsync(request);

            var result = new WebClientByteResult();
            result.Content = await response.Content.ReadAsByteArrayAsync();

            // some cloudflare clients are using a refresh header
            // Pull it out manually 
            if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
            {
                var refreshHeaders = response.Headers.GetValues("Refresh");
                var redirval = "";
                var redirtime = 0;
                if (refreshHeaders != null)
                {
                    foreach (var value in refreshHeaders)
                    {
                        var start = value.IndexOf("=");
                        var end = value.IndexOf(";");
                        var len = value.Length;
                        if (start > -1)
                        {
                            redirval = value.Substring(start + 1);
                            result.RedirectingTo = redirval;
                            // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                            // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                            // it shoudln't include service unavailable..only if we have this redirect header.
                            response.StatusCode = System.Net.HttpStatusCode.Redirect;
                            redirtime = Int32.Parse(value.Substring(0, end));
                            System.Threading.Thread.Sleep(redirtime * 1000);
                        }
                    }
                }
            }
            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable<string> cookieHeaders;
            var responseCookies = new List<Tuple<string, string>>();

            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                foreach (var value in cookieHeaders)
                {
                    var nameSplit = value.IndexOf('=');
                    if (nameSplit > -1)
                    {
                        responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
                    }
                }

                var cookieBuilder = new StringBuilder();
                foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                {
                    cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                }
                result.Cookies = cookieBuilder.ToString().Trim();
            }

            ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
            return result;
        }
Пример #21
0
        private async Task <WebClientByteResult> Run(WebRequest request)
        {
            var args = new StringBuilder();

            args.AppendFormat("--url \"{0}\" ", request.Url);

            if (request.EmulateBrowser)
            {
                args.AppendFormat("-i  -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
            }
            else
            {
                args.AppendFormat("-i  -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());
            }

            if (!string.IsNullOrWhiteSpace(request.Cookies))
            {
                args.AppendFormat("--cookie \"{0}\" ", request.Cookies);
            }

            if (!string.IsNullOrWhiteSpace(request.Referer))
            {
                args.AppendFormat("--referer \"{0}\" ", request.Referer);
            }

            if (!string.IsNullOrEmpty(request.RawBody))
            {
                var postString = StringUtil.PostDataFromDict(request.PostData);
                args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
            }
            else if (request.PostData != null && request.PostData.Count() > 0)
            {
                var postString = StringUtil.PostDataFromDict(request.PostData);
                args.AppendFormat("--data \"{0}\" ", postString);
            }

            var tempFile = Path.GetTempFileName();

            args.AppendFormat("--output \"{0}\" ", tempFile);

            if (Startup.DoSSLFix == true)
            {
                // http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
                // https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
                args.Append("--cipher " + SSLFix.CipherList);
            }

            string stdout = null;
            await Task.Run(() =>
            {
                stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
            });

            var outputData = File.ReadAllBytes(tempFile);

            File.Delete(tempFile);
            stdout = Encoding.UTF8.GetString(outputData);
            var result    = new WebClientByteResult();
            var headSplit = stdout.IndexOf("\r\n\r\n");

            if (headSplit < 0)
            {
                throw new Exception("Invalid response");
            }
            var headers       = stdout.Substring(0, headSplit);
            var headerCount   = 0;
            var cookieBuilder = new StringBuilder();
            var cookies       = new List <Tuple <string, string> >();

            foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
            {
                if (headerCount == 0)
                {
                    var responseCode = int.Parse(header.Split(' ')[1]);
                    result.Status = (HttpStatusCode)responseCode;
                }
                else
                {
                    var headerSplitIndex = header.IndexOf(':');
                    if (headerSplitIndex > 0)
                    {
                        var name  = header.Substring(0, headerSplitIndex).ToLowerInvariant();
                        var value = header.Substring(headerSplitIndex + 1);
                        switch (name)
                        {
                        case "set-cookie":
                            var nameSplit = value.IndexOf('=');
                            if (nameSplit > -1)
                            {
                                cookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
                            }
                            break;

                        case "location":
                            result.RedirectingTo = value.Trim();
                            break;
                        }
                    }
                }
                headerCount++;
            }

            foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
            {
                cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
            }

            result.Cookies = cookieBuilder.ToString().Trim();
            result.Content = new byte[outputData.Length - (headSplit + 3)];
            var dest = 0;

            for (int i = headSplit + 4; i < outputData.Length; i++)
            {
                result.Content[dest] = outputData[i];
                dest++;
            }

            logger.Debug("WebClientByteResult returned " + result.Status);
            ServerUtil.ResureRedirectIsFullyQualified(request, result);
            return(result);
        }
Пример #22
0
 protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
 {
     var byteResult = new WebClientByteResult();
     // Map to byte
     Mapper.Map(response, byteResult);
     await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies);
     // Map to string
     Mapper.Map(byteResult, response);
 }
Пример #23
0
        private async Task <WebClientByteResult> Run(WebRequest webRequest)
        {
            ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;

            var cookies = new CookieContainer();

            if (!string.IsNullOrEmpty(webRequest.Cookies))
            {
                var uri = new Uri(webRequest.Url);
                foreach (var c in webRequest.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(uri, c);
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }
            var      useProxy    = false;
            WebProxy proxyServer = null;

            if (Startup.ProxyConnection != null)
            {
                proxyServer = new WebProxy(Startup.ProxyConnection, false);
                useProxy    = true;
            }

            ClearanceHandler  clearanceHandlr = new ClearanceHandler();
            HttpClientHandler clientHandlr    = new HttpClientHandler
            {
                CookieContainer   = cookies,
                AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                UseCookies        = true,
                Proxy             = proxyServer,
                UseProxy          = useProxy
            };

            clearanceHandlr.InnerHandler = clientHandlr;
            var client = new HttpClient(clearanceHandlr);

            if (webRequest.EmulateBrowser)
            {
                client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
            }
            else
            {
                client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
            }

            HttpResponseMessage response = null;
            var request = new HttpRequestMessage();

            request.Headers.ExpectContinue = false;
            request.RequestUri             = new Uri(webRequest.Url);

            if (webRequest.Headers != null)
            {
                foreach (var header in webRequest.Headers)
                {
                    if (header.Key != "Content-Type")
                    {
                        request.Headers.Add(header.Key, header.Value);
                    }
                }
            }

            if (!string.IsNullOrEmpty(webRequest.RawBody))
            {
                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
                if (type.HasValue)
                {
                    var str = new StringContent(webRequest.RawBody);
                    str.Headers.Remove("Content-Type");
                    str.Headers.Add("Content-Type", type.Value.Value);
                    request.Content = str;
                }
                else
                {
                    request.Content = new StringContent(webRequest.RawBody);
                }
                request.Method = HttpMethod.Post;
            }
            else if (webRequest.Type == RequestType.POST)
            {
                request.Content = new FormUrlEncodedContent(webRequest.PostData);
                request.Method  = HttpMethod.Post;
            }
            else
            {
                request.Method = HttpMethod.Get;
            }

            response = await client.SendAsync(request);

            var result = new WebClientByteResult();

            result.Content = await response.Content.ReadAsByteArrayAsync();

            // some cloudflare clients are using a refresh header
            // Pull it out manually
            if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
            {
                var refreshHeaders = response.Headers.GetValues("Refresh");
                var redirval       = "";
                var redirtime      = 0;
                if (refreshHeaders != null)
                {
                    foreach (var value in refreshHeaders)
                    {
                        var start = value.IndexOf("=");
                        var end   = value.IndexOf(";");
                        var len   = value.Length;
                        if (start > -1)
                        {
                            redirval             = value.Substring(start + 1);
                            result.RedirectingTo = redirval;
                            // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                            // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                            // it shoudln't include service unavailable..only if we have this redirect header.
                            response.StatusCode = System.Net.HttpStatusCode.Redirect;
                            redirtime           = Int32.Parse(value.Substring(0, end));
                            System.Threading.Thread.Sleep(redirtime * 1000);
                        }
                    }
                }
            }
            if (response.Headers.Location != null)
            {
                result.RedirectingTo = response.Headers.Location.ToString();
            }
            result.Status = response.StatusCode;

            // Compatiblity issue between the cookie format and httpclient
            // Pull it out manually ignoring the expiry date then set it manually
            // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
            IEnumerable <string> cookieHeaders;
            var responseCookies = new List <Tuple <string, string> >();

            if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
            {
                foreach (var value in cookieHeaders)
                {
                    var nameSplit = value.IndexOf('=');
                    if (nameSplit > -1)
                    {
                        responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';') + 1))));
                    }
                }

                var cookieBuilder = new StringBuilder();
                foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                {
                    cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                }
                result.Cookies = cookieBuilder.ToString().Trim();
            }

            ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
            return(result);
        }
Пример #24
0
 private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null)
 {
     if (incomingResponse.IsRedirect)
     {
         // Do redirect
         var redirectedResponse = await webclient.GetBytes(new WebRequest()
         {
             Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo,
             Referer = referrer,
             Cookies = overrideCookies ?? CookieHeader
         });
         Mapper.Map(redirectedResponse, incomingResponse);
     }
 }
Пример #25
0
        override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
        {
            ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;

            var cookies = new CookieContainer();

            if (!string.IsNullOrEmpty(webRequest.Cookies))
            {
                var uri       = new Uri(webRequest.Url);
                var cookieUrl = new Uri(uri.Scheme + "://" + uri.Host); // don't include the path, Scheme is needed for mono compatibility
                foreach (var c in webRequest.Cookies.Split(';'))
                {
                    try
                    {
                        cookies.SetCookies(cookieUrl, c.Trim());
                    }
                    catch (CookieException ex)
                    {
                        logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
                    }
                }
            }
            var useProxy = false;

            using (ClearanceHandler clearanceHandlr = new ClearanceHandler())
            {
                IWebProxy proxyServer = null;
                var       proxyUrl    = serverConfig.GetProxyUrl();
                if (!string.IsNullOrWhiteSpace(proxyUrl))
                {
                    useProxy = true;
                    NetworkCredential creds = null;
                    if (!serverConfig.ProxyIsAnonymous)
                    {
                        var username = serverConfig.ProxyUsername;
                        var password = serverConfig.ProxyPassword;
                        creds = new NetworkCredential(username, password);
                    }
                    if (serverConfig.ProxyType != ProxyType.Http)
                    {
                        var addresses = await Dns.GetHostAddressesAsync(serverConfig.ProxyUrl);

                        var socksConfig = new ProxyConfig
                        {
                            SocksAddress = addresses.FirstOrDefault(),
                            Username     = serverConfig.ProxyUsername,
                            Password     = serverConfig.ProxyPassword,
                            Version      = serverConfig.ProxyType == ProxyType.Socks4 ?
                                           ProxyConfig.SocksVersion.Four :
                                           ProxyConfig.SocksVersion.Five
                        };
                        if (serverConfig.ProxyPort.HasValue)
                        {
                            socksConfig.SocksPort = serverConfig.ProxyPort.Value;
                        }
                        proxyServer = new SocksWebProxy(socksConfig, false);
                    }
                    else
                    {
                        proxyServer = new WebProxy(proxyUrl)
                        {
                            BypassProxyOnLocal = false,
                            Credentials        = creds
                        };
                    }
                }
                using (HttpClientHandler clientHandlr = new HttpClientHandler
                {
                    CookieContainer = cookies,
                    AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
                    UseCookies = true,
                    Proxy = proxyServer,
                    UseProxy = useProxy,
                    AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
                })
                {
                    clearanceHandlr.InnerHandler = clientHandlr;
                    using (var client = new HttpClient(clearanceHandlr))
                    {
                        if (webRequest.EmulateBrowser == true)
                        {
                            client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
                        }
                        else
                        {
                            client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
                        }

                        HttpResponseMessage response = null;
                        using (var request = new HttpRequestMessage())
                        {
                            request.Headers.ExpectContinue = false;
                            request.RequestUri             = new Uri(webRequest.Url);

                            if (webRequest.Headers != null)
                            {
                                foreach (var header in webRequest.Headers)
                                {
                                    if (header.Key != "Content-Type")
                                    {
                                        request.Headers.TryAddWithoutValidation(header.Key, header.Value);
                                    }
                                }
                            }

                            if (!string.IsNullOrEmpty(webRequest.Referer))
                            {
                                request.Headers.Referrer = new Uri(webRequest.Referer);
                            }

                            if (!string.IsNullOrEmpty(webRequest.RawBody))
                            {
                                var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
                                if (type.HasValue)
                                {
                                    var str = new StringContent(webRequest.RawBody);
                                    str.Headers.Remove("Content-Type");
                                    str.Headers.Add("Content-Type", type.Value.Value);
                                    request.Content = str;
                                }
                                else
                                {
                                    request.Content = new StringContent(webRequest.RawBody);
                                }
                                request.Method = HttpMethod.Post;
                            }
                            else if (webRequest.Type == RequestType.POST)
                            {
                                if (webRequest.PostData != null)
                                {
                                    request.Content = new FormUrlEncodedContent(webRequest.PostData);
                                }
                                request.Method = HttpMethod.Post;
                            }
                            else
                            {
                                request.Method = HttpMethod.Get;
                            }

                            using (response = await client.SendAsync(request))
                            {
                                var result = new WebClientByteResult
                                {
                                    Content = await response.Content.ReadAsByteArrayAsync()
                                };

                                foreach (var header in response.Headers)
                                {
                                    IEnumerable <string> value = header.Value;
                                    result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
                                }

                                // some cloudflare clients are using a refresh header
                                // Pull it out manually
                                if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
                                {
                                    var refreshHeaders = response.Headers.GetValues("Refresh");
                                    var redirval       = "";
                                    var redirtime      = 0;
                                    if (refreshHeaders != null)
                                    {
                                        foreach (var value in refreshHeaders)
                                        {
                                            var start = value.IndexOf("=");
                                            var end   = value.IndexOf(";");
                                            var len   = value.Length;
                                            if (start > -1)
                                            {
                                                redirval             = value.Substring(start + 1);
                                                result.RedirectingTo = redirval;
                                                // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
                                                // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
                                                // it shoudln't include service unavailable..only if we have this redirect header.
                                                response.StatusCode = System.Net.HttpStatusCode.Redirect;
                                                redirtime           = Int32.Parse(value.Substring(0, end));
                                                System.Threading.Thread.Sleep(redirtime * 1000);
                                            }
                                        }
                                    }
                                }
                                if (response.Headers.Location != null)
                                {
                                    result.RedirectingTo = response.Headers.Location.ToString();
                                }
                                // Mono won't add the baseurl to relative redirects.
                                // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
                                // See issue #1200
                                if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
                                {
                                    var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
                                    logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
                                    result.RedirectingTo = newRedirectingTo;
                                }
                                result.Status = response.StatusCode;

                                // Compatiblity issue between the cookie format and httpclient
                                // Pull it out manually ignoring the expiry date then set it manually
                                // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
                                IEnumerable <string> cookieHeaders;
                                var responseCookies = new List <Tuple <string, string> >();

                                if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
                                {
                                    foreach (var value in cookieHeaders)
                                    {
                                        var nameSplit = value.IndexOf('=');
                                        if (nameSplit > -1)
                                        {
                                            responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
                                        }
                                    }

                                    var cookieBuilder = new StringBuilder();
                                    foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
                                    {
                                        cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
                                    }
                                    result.Cookies = cookieBuilder.ToString().Trim();
                                }
                                ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
                                return(result);
                            }
                        }
                    }
                }
            }
        }