예제 #1
0
        public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified()
        {
            var res = new WebClientByteResult()
            {
                RedirectingTo = "list?p=1"
            };

            var req = new WebRequest()
            {
                Url = "http://my.domain.com/page.php"
            };

            // Not fully qualified  requiring redirect
            ServerUtil.ResureRedirectIsFullyQualified(req, res);
            Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1");

            // Fully qualified not needing modified
            res.RedirectingTo = "http://a.domain/page.htm";
            ServerUtil.ResureRedirectIsFullyQualified(req, res);
            Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm");

            // Relative  requiring redirect
            req.Url = "http://my.domain.com/dir/page.php";
            res.RedirectingTo = "a/dir/page.html";
            ServerUtil.ResureRedirectIsFullyQualified(req, res);
            Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html");
        }
예제 #2
0
 virtual public async Task<WebClientByteResult> GetBytes(WebRequest request)
 {
     logger.Debug(string.Format("IWebClient.GetBytes(Url:{0})", request.Url));
     var result = await Run(request);
     logger.Debug(string.Format("IWebClient: Returning {0} => {1} bytes", result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (result.Content == null ? "<NULL>" : result.Content.Length.ToString())));
     return result;
 }
예제 #3
0
        virtual public async Task<WebClientStringResult> GetString(WebRequest request)
        {
            logger.Debug(string.Format("IWebClient.GetString(Url:{0})", request.Url));
            var result = await Run(request);
            WebClientStringResult stringResult = Mapper.Map<WebClientStringResult>(result);
            Encoding encoding = null;
            if (request.Encoding != null)
            {
                encoding = request.Encoding;
            }
            else if (result.Headers.ContainsKey("content-type"))
            {
                Regex CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled);
                var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]);
                if (CharsetRegexMatch.Success)
                {
                    var charset = CharsetRegexMatch.Groups[1].Value;
                    try
                    {
                        encoding = Encoding.GetEncoding(charset);
                    }
                    catch (Exception ex)
                    {
                        logger.Error(string.Format("IWebClient.GetString(Url:{0}): Error loading encoding {0} based on header {1}: {2}", request.Url, charset, result.Headers["content-type"][0], ex));
                    }
                }
                else
                {
                    logger.Error(string.Format("IWebClient.GetString(Url:{0}): Got header without charset: {0}", request.Url, result.Headers["content-type"][0]));
                }
            }

            if (encoding == null)
            {
                logger.Error(string.Format("IWebClient.GetString(Url:{0}): No encoding detected, defaulting to UTF-8", request.Url));
                encoding = Encoding.UTF8;
            }

            string decodedContent = null;
            if (result.Content != null)
                decodedContent = encoding.GetString(result.Content);

            stringResult.Content = decodedContent;
            logger.Debug(string.Format("IWebClient: Returning {0} => {1}", result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (decodedContent == null ? "<NULL>" : decodedContent)));

            string[] server;
            if (stringResult.Headers.TryGetValue("server", out server))
            {
                if (server[0] == "cloudflare-nginx")
                    stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content);
            }
            return stringResult;
        }
예제 #4
0
        public async Task<WebClientStringResult> GetString(WebRequest request)
        {
            logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url));
            var result = await GetBytes(request);

            var sresult = new WebClientStringResult()
            {
                Content = Encoding.UTF8.GetString(result.Content),
                Cookies = result.Cookies,
                Status = result.Status
            };

            return sresult;
        }
예제 #5
0
        public async Task<WebClientByteResult> GetBytes(WebRequest request)
        {
            Jackett.CurlHelper.CurlResponse response;

            logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url));

            if (request.Type == RequestType.GET)
            {
                response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
            }
            else
            {
                response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
            }

            var result = new WebClientByteResult()
            {
                Content = response.Content,
                Cookies = response.CookieHeader,
                Status = response.Status
            };

            if (response.Headers != null)
            {
                foreach(var header in response.Headers)
                {
                    if(string.Equals(header.Key, "location", StringComparison.InvariantCultureIgnoreCase) && header.Value !=null)
                    {
                        result.RedirectingTo = header.Value;
                    }
                }
            }

            logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status));
            return result;
        }
예제 #6
0
 public Task<WebClientStringResult> GetString(WebRequest request)
 {
     return Task.FromResult<WebClientStringResult>(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
 }
예제 #7
0
 public Task<WebClientByteResult> GetBytes(WebRequest request)
 {
    return Task.FromResult< WebClientByteResult>(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
 }
예제 #8
0
 public void RegisterStringCall(WebRequest req, Func<WebRequest, WebClientStringResult> f)
 {
     stringCallbacks.Add(req, f);
 }
예제 #9
0
 public void RegisterByteCall(WebRequest req, Func<WebRequest, WebClientByteResult> f)
 {
     byteCallbacks.Add(req, f);
 }
예제 #10
0
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
        virtual protected async Task<WebClientByteResult> Run(WebRequest webRequest) { throw new NotImplementedException(); }
예제 #11
0
 public static void ResureRedirectIsFullyQualified(WebRequest req, BaseWebResult result)
 {
     if (!string.IsNullOrEmpty(result.RedirectingTo))
     {
         var destLower = result.RedirectingTo.ToLowerInvariant();
         if (!destLower.StartsWith("http"))
         {
             var hostUri = new Uri(req.Url);
             var fullUri = new Uri(hostUri, result.RedirectingTo);
             result.RedirectingTo = fullUri.ToString();
         }
     }
 }
예제 #12
0
        private WebRequest SetDownloadHeaders(WebRequest req, string token)
        {
            req.Headers = new Dictionary<string, string>()
            {
                { "Accept", "application/octet-stream" }
            };

            if (!string.IsNullOrWhiteSpace(token))
            {
                req.Headers.Add("Authorization", "token " + token);
            }

            return req;
        }
예제 #13
0
파일: Xthor.cs 프로젝트: CatmanIX/Shades
        /// <summary>
        /// Get Torrents Page from Tracker by Query Provided
        /// </summary>
        /// <param name="request">URL created by Query Builder</param>
        /// <returns>Results from query</returns>
        private async Task<WebClientStringResult> QueryTracker(string request)
        {
            // Cache mode not enabled or cached file didn't exist for our query
            Output("\nQuerying tracker for results....");

            // Build WebRequest for index
            var myIndexRequest = new WebRequest()
            {
                Type = RequestType.GET,
                Url = request,
                Headers = EmulatedBrowserHeaders
            };

            // Request our first page
            var results = await webclient.GetString(myIndexRequest);

            // Return results from tracker
            return results;
        }
예제 #14
0
        /// <summary>
        /// Download torrent file from tracker
        /// </summary>
        /// <param name="link">URL string</param>
        /// <returns></returns>
        public override async Task<byte[]> Download(Uri link)
        {
            // This tracker need to thanks Uploader before getting torrent file...
            Output("\nThis tracker needs you to thank uploader before downloading torrent!");

            // Retrieving ID from link provided
            var id = ParseUtil.CoerceInt(Regex.Match(link.AbsoluteUri, @"\d+").Value);
            Output("Torrent Requested ID: " + id);

            // Building login form data
            var pairs = new Dictionary<string, string> {
                { "torrentid", id.ToString() },
                { "_", string.Empty } // ~~ Strange, blank param...
            };

            // Add emulated XHR request
            _emulatedBrowserHeaders.Add("X-Prototype-Version", "1.6.0.3");
            _emulatedBrowserHeaders.Add("X-Requested-With", "XMLHttpRequest");

            // Build WebRequest for thanks
            var myRequestThanks = new WebRequest()
            {
                Type = RequestType.POST,
                PostData = pairs,
                Url = TorrentThanksUrl,
                Headers = _emulatedBrowserHeaders,
                Cookies = ConfigData.CookieHeader.Value,
                Referer = TorrentDescriptionUrl.Replace("{id}", id.ToString())
            };

            // Get thanks page -- (not used, just for doing a request)
            LatencyNow();
            Output("Thanks user, to get download link for our torrent.. with " + TorrentThanksUrl);
            await webclient.GetString(myRequestThanks);

            // Get torrent file now
            Output("Getting torrent file now....");
            var response = await base.Download(link);

            // Remove our XHR request header
            _emulatedBrowserHeaders.Remove("X-Prototype-Version");
            _emulatedBrowserHeaders.Remove("X-Requested-With");

            // Return content
            return response;
        }
예제 #15
0
        private WebRequest SetDownloadHeaders(WebRequest req)
        {
            req.Headers = new Dictionary<string, string>()
            {
                { "Accept", "application/octet-stream" }
            };

            return req;
        }
예제 #16
0
파일: Norbits.cs 프로젝트: Jackett/Jackett
        /// <summary>
        /// Perform login to racker
        /// </summary>
        /// <returns></returns>
        private async Task DoLogin()
        {
            // Build WebRequest for index
            var myIndexRequest = new WebRequest()
            {
                Type = RequestType.GET,
                Url = SiteLink,
                Headers = _emulatedBrowserHeaders,
                Encoding = Encoding
            };

            // Get index page for cookies
            Output("\nGetting index page (for cookies).. with " + SiteLink);
            var indexPage = await webclient.GetString(myIndexRequest);

            // Building login form data
            var pairs = new Dictionary<string, string> {
                { "username", ConfigData.Username.Value },
                { "password", ConfigData.Password.Value }
            };

            // Build WebRequest for login
            var myRequestLogin = new WebRequest()
            {
                Type = RequestType.GET,
                Url = LoginUrl,
                Headers = _emulatedBrowserHeaders,
                Cookies = indexPage.Cookies,
                Referer = SiteLink,
                Encoding = Encoding
            };

            // Get login page -- (not used, but simulation needed by tracker security's checks)
            LatencyNow();
            Output("\nGetting login page (user simulation).. with " + LoginUrl);
            await webclient.GetString(myRequestLogin);

            // Build WebRequest for submitting authentification
            var request = new WebRequest()
            {
                PostData = pairs,
                Referer = LoginUrl,
                Type = RequestType.POST,
                Url = LoginCheckUrl,
                Headers = _emulatedBrowserHeaders,
                Cookies = indexPage.Cookies,
                Encoding = Encoding
            };

            // Perform loggin
            LatencyNow();
            Output("\nPerform loggin.. with " + LoginCheckUrl);
            var response = await webclient.GetString(request);

            // Test if we are logged in
            await ConfigureIfOK(response.Cookies, response.Content != null && response.Cookies.Contains("uid="), () =>
            {
                // Default error message
                var message = "Error during attempt !";
                // Parse redirect header
                var redirectTo = response.RedirectingTo;
                
                // Oops, unable to login
                Output("-> Login failed: " + message, "error");
                throw new ExceptionWithConfigData("Login failed: " + message, configData);
            });

            Output("\nCookies saved for future uses...");
            ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username="******"\n-> Login Success\n");
        }
예제 #17
0
 public static void RegisterStringCall(WebRequest r, Func<WebRequest, WebClientStringResult> f)
 {
     var client = testContainer.Resolve<IWebClient>() as TestWebClient;
     client.RegisterStringCall(r, f);
 }
예제 #18
0
        /// <summary>
        /// Configure our FADN Provider
        /// </summary>
        /// <param name="configJson">Our params in Json</param>
        /// <returns>Configuration state</returns>
        public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
        {
            // Retrieve config values set by Jackett's user
            ConfigData.LoadValuesFromJson(configJson);

            // Check & Validate Config
            validateConfig();

            // Setting our data for a better emulated browser (maximum security)
            // TODO: Encoded Content not supported by Jackett at this time
            // emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate");

            // If we want to simulate a browser
            if (ConfigData.Browser.Value)
            {

                // Clean headers
                emulatedBrowserHeaders.Clear();

                // Inject headers
                emulatedBrowserHeaders.Add("Accept", ConfigData.HeaderAccept.Value);
                emulatedBrowserHeaders.Add("Accept-Language", ConfigData.HeaderAcceptLang.Value);
                emulatedBrowserHeaders.Add("DNT", Convert.ToInt32(ConfigData.HeaderDNT.Value).ToString());
                emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", Convert.ToInt32(ConfigData.HeaderUpgradeInsecure.Value).ToString());
                emulatedBrowserHeaders.Add("User-Agent", ConfigData.HeaderUserAgent.Value);
            }

            // Build WebRequest for index
            var myIndexRequest = new WebRequest()
            {
                Type = RequestType.GET,
                Url = SiteLink,
                Headers = emulatedBrowserHeaders
            };

            // Get index page for cookies
            output("\nGetting index page (for cookies).. with " + SiteLink);
            var indexPage = await webclient.GetString(myIndexRequest);

             // Building login form data
             var pairs = new Dictionary<string, string> {
                { "username", ConfigData.Username.Value },
                { "password", ConfigData.Password.Value }
            };

            // Build WebRequest for login
            var myRequestLogin = new WebRequest()
            {
                Type = RequestType.GET,
                Url = LoginUrl,
                Headers = emulatedBrowserHeaders,
                Cookies = indexPage.Cookies,
                Referer = SiteLink
            };

            // Get login page -- (not used, but simulation needed by tracker security's checks)
            latencyNow();
            output("\nGetting login page (user simulation).. with " + LoginUrl);
            var loginPage = await webclient.GetString(myRequestLogin);

            // Build WebRequest for submitting authentification
            var request = new WebRequest()
            {
                PostData = pairs,
                Referer = LoginUrl,
                Type = RequestType.POST,
                Url = LoginCheckUrl,
                Headers = emulatedBrowserHeaders,
                Cookies = indexPage.Cookies,
   
            };

            // Perform loggin
            latencyNow();
            output("\nPerform loggin.. with " + LoginCheckUrl);
            var response = await webclient.GetString(request);

            // Test if we are logged in
            await ConfigureIfOK(response.Cookies, !string.IsNullOrEmpty(response.Cookies) && !response.IsRedirect, () =>
            {
                // Default error message
                string message = "Error during attempt !";

                // Parse redirect header
                string redirectTo = response.RedirectingTo;

                // Analyzer error code
                if(redirectTo.Contains("login.php?error=4"))
                {
                    // Set message
                    message = "Wrong username or password !";
                }

                // Oops, unable to login
                output("-> Login failed: " + message, "error");
                throw new ExceptionWithConfigData("Login failed: " + message, configData);
            });

            output("\nCookies saved for future uses...");
            ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username="******"\n-> Login Success\n");

            return IndexerConfigurationStatus.RequiresTesting;
        }