public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified()
{
var res = new WebClientByteResult()
{
RedirectingTo = "list?p=1"
};
var req = new WebRequest()
{
Url = "http://my.domain.com/page.php"
};
// Not fully qualified requiring redirect
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1");
// Fully qualified not needing modified
res.RedirectingTo = "http://a.domain/page.htm";
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm");
// Relative requiring redirect
req.Url = "http://my.domain.com/dir/page.php";
res.RedirectingTo = "a/dir/page.html";
ServerUtil.ResureRedirectIsFullyQualified(req, res);
Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html");
}
virtual public async Task<WebClientByteResult> GetBytes(WebRequest request)
{
logger.Debug(string.Format("IWebClient.GetBytes(Url:{0})", request.Url));
var result = await Run(request);
logger.Debug(string.Format("IWebClient: Returning {0} => {1} bytes", result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (result.Content == null ? "<NULL>" : result.Content.Length.ToString())));
return result;
}
virtual public async Task<WebClientStringResult> GetString(WebRequest request)
{
logger.Debug(string.Format("IWebClient.GetString(Url:{0})", request.Url));
var result = await Run(request);
WebClientStringResult stringResult = Mapper.Map<WebClientStringResult>(result);
Encoding encoding = null;
if (request.Encoding != null)
{
encoding = request.Encoding;
}
else if (result.Headers.ContainsKey("content-type"))
{
Regex CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled);
var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]);
if (CharsetRegexMatch.Success)
{
var charset = CharsetRegexMatch.Groups[1].Value;
try
{
encoding = Encoding.GetEncoding(charset);
}
catch (Exception ex)
{
logger.Error(string.Format("IWebClient.GetString(Url:{0}): Error loading encoding {0} based on header {1}: {2}", request.Url, charset, result.Headers["content-type"][0], ex));
}
}
else
{
logger.Error(string.Format("IWebClient.GetString(Url:{0}): Got header without charset: {0}", request.Url, result.Headers["content-type"][0]));
}
}
if (encoding == null)
{
logger.Error(string.Format("IWebClient.GetString(Url:{0}): No encoding detected, defaulting to UTF-8", request.Url));
encoding = Encoding.UTF8;
}
string decodedContent = null;
if (result.Content != null)
decodedContent = encoding.GetString(result.Content);
stringResult.Content = decodedContent;
logger.Debug(string.Format("IWebClient: Returning {0} => {1}", result.Status, (result.IsRedirect ? result.RedirectingTo + " " : "") + (decodedContent == null ? "<NULL>" : decodedContent)));
string[] server;
if (stringResult.Headers.TryGetValue("server", out server))
{
if (server[0] == "cloudflare-nginx")
stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content);
}
return stringResult;
}
public async Task<WebClientStringResult> GetString(WebRequest request)
{
logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url));
var result = await GetBytes(request);
var sresult = new WebClientStringResult()
{
Content = Encoding.UTF8.GetString(result.Content),
Cookies = result.Cookies,
Status = result.Status
};
return sresult;
}
public async Task<WebClientByteResult> GetBytes(WebRequest request)
{
Jackett.CurlHelper.CurlResponse response;
logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url));
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer);
}
else
{
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
}
var result = new WebClientByteResult()
{
Content = response.Content,
Cookies = response.CookieHeader,
Status = response.Status
};
if (response.Headers != null)
{
foreach(var header in response.Headers)
{
if(string.Equals(header.Key, "location", StringComparison.InvariantCultureIgnoreCase) && header.Value !=null)
{
result.RedirectingTo = header.Value;
}
}
}
logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status));
return result;
}
public Task<WebClientStringResult> GetString(WebRequest request)
{
return Task.FromResult<WebClientStringResult>(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
}
public Task<WebClientByteResult> GetBytes(WebRequest request)
{
return Task.FromResult< WebClientByteResult>(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request));
}
public void RegisterStringCall(WebRequest req, Func<WebRequest, WebClientStringResult> f)
{
stringCallbacks.Add(req, f);
}
public void RegisterByteCall(WebRequest req, Func<WebRequest, WebClientByteResult> f)
{
byteCallbacks.Add(req, f);
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
virtual protected async Task<WebClientByteResult> Run(WebRequest webRequest) { throw new NotImplementedException(); }
public static void ResureRedirectIsFullyQualified(WebRequest req, BaseWebResult result)
{
if (!string.IsNullOrEmpty(result.RedirectingTo))
{
var destLower = result.RedirectingTo.ToLowerInvariant();
if (!destLower.StartsWith("http"))
{
var hostUri = new Uri(req.Url);
var fullUri = new Uri(hostUri, result.RedirectingTo);
result.RedirectingTo = fullUri.ToString();
}
}
}
private WebRequest SetDownloadHeaders(WebRequest req, string token)
{
req.Headers = new Dictionary<string, string>()
{
{ "Accept", "application/octet-stream" }
};
if (!string.IsNullOrWhiteSpace(token))
{
req.Headers.Add("Authorization", "token " + token);
}
return req;
}
/// <summary>
/// Get Torrents Page from Tracker by Query Provided
/// </summary>
/// <param name="request">URL created by Query Builder</param>
/// <returns>Results from query</returns>
private async Task<WebClientStringResult> QueryTracker(string request)
{
// Cache mode not enabled or cached file didn't exist for our query
Output("\nQuerying tracker for results....");
// Build WebRequest for index
var myIndexRequest = new WebRequest()
{
Type = RequestType.GET,
Url = request,
Headers = EmulatedBrowserHeaders
};
// Request our first page
var results = await webclient.GetString(myIndexRequest);
// Return results from tracker
return results;
}
/// <summary>
/// Download torrent file from tracker
/// </summary>
/// <param name="link">URL string</param>
/// <returns></returns>
public override async Task<byte[]> Download(Uri link)
{
// This tracker need to thanks Uploader before getting torrent file...
Output("\nThis tracker needs you to thank uploader before downloading torrent!");
// Retrieving ID from link provided
var id = ParseUtil.CoerceInt(Regex.Match(link.AbsoluteUri, @"\d+").Value);
Output("Torrent Requested ID: " + id);
// Building login form data
var pairs = new Dictionary<string, string> {
{ "torrentid", id.ToString() },
{ "_", string.Empty } // ~~ Strange, blank param...
};
// Add emulated XHR request
_emulatedBrowserHeaders.Add("X-Prototype-Version", "1.6.0.3");
_emulatedBrowserHeaders.Add("X-Requested-With", "XMLHttpRequest");
// Build WebRequest for thanks
var myRequestThanks = new WebRequest()
{
Type = RequestType.POST,
PostData = pairs,
Url = TorrentThanksUrl,
Headers = _emulatedBrowserHeaders,
Cookies = ConfigData.CookieHeader.Value,
Referer = TorrentDescriptionUrl.Replace("{id}", id.ToString())
};
// Get thanks page -- (not used, just for doing a request)
LatencyNow();
Output("Thanks user, to get download link for our torrent.. with " + TorrentThanksUrl);
await webclient.GetString(myRequestThanks);
// Get torrent file now
Output("Getting torrent file now....");
var response = await base.Download(link);
// Remove our XHR request header
_emulatedBrowserHeaders.Remove("X-Prototype-Version");
_emulatedBrowserHeaders.Remove("X-Requested-With");
// Return content
return response;
}
private WebRequest SetDownloadHeaders(WebRequest req)
{
req.Headers = new Dictionary<string, string>()
{
{ "Accept", "application/octet-stream" }
};
return req;
}
/// <summary>
/// Perform login to racker
/// </summary>
/// <returns></returns>
private async Task DoLogin()
{
// Build WebRequest for index
var myIndexRequest = new WebRequest()
{
Type = RequestType.GET,
Url = SiteLink,
Headers = _emulatedBrowserHeaders,
Encoding = Encoding
};
// Get index page for cookies
Output("\nGetting index page (for cookies).. with " + SiteLink);
var indexPage = await webclient.GetString(myIndexRequest);
// Building login form data
var pairs = new Dictionary<string, string> {
{ "username", ConfigData.Username.Value },
{ "password", ConfigData.Password.Value }
};
// Build WebRequest for login
var myRequestLogin = new WebRequest()
{
Type = RequestType.GET,
Url = LoginUrl,
Headers = _emulatedBrowserHeaders,
Cookies = indexPage.Cookies,
Referer = SiteLink,
Encoding = Encoding
};
// Get login page -- (not used, but simulation needed by tracker security's checks)
LatencyNow();
Output("\nGetting login page (user simulation).. with " + LoginUrl);
await webclient.GetString(myRequestLogin);
// Build WebRequest for submitting authentification
var request = new WebRequest()
{
PostData = pairs,
Referer = LoginUrl,
Type = RequestType.POST,
Url = LoginCheckUrl,
Headers = _emulatedBrowserHeaders,
Cookies = indexPage.Cookies,
Encoding = Encoding
};
// Perform loggin
LatencyNow();
Output("\nPerform loggin.. with " + LoginCheckUrl);
var response = await webclient.GetString(request);
// Test if we are logged in
await ConfigureIfOK(response.Cookies, response.Content != null && response.Cookies.Contains("uid="), () =>
{
// Default error message
var message = "Error during attempt !";
// Parse redirect header
var redirectTo = response.RedirectingTo;
// Oops, unable to login
Output("-> Login failed: " + message, "error");
throw new ExceptionWithConfigData("Login failed: " + message, configData);
});
Output("\nCookies saved for future uses...");
ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username="******"\n-> Login Success\n");
}
public static void RegisterStringCall(WebRequest r, Func<WebRequest, WebClientStringResult> f)
{
var client = testContainer.Resolve<IWebClient>() as TestWebClient;
client.RegisterStringCall(r, f);
}
/// <summary>
/// Configure our FADN Provider
/// </summary>
/// <param name="configJson">Our params in Json</param>
/// <returns>Configuration state</returns>
public async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
// Retrieve config values set by Jackett's user
ConfigData.LoadValuesFromJson(configJson);
// Check & Validate Config
validateConfig();
// Setting our data for a better emulated browser (maximum security)
// TODO: Encoded Content not supported by Jackett at this time
// emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate");
// If we want to simulate a browser
if (ConfigData.Browser.Value)
{
// Clean headers
emulatedBrowserHeaders.Clear();
// Inject headers
emulatedBrowserHeaders.Add("Accept", ConfigData.HeaderAccept.Value);
emulatedBrowserHeaders.Add("Accept-Language", ConfigData.HeaderAcceptLang.Value);
emulatedBrowserHeaders.Add("DNT", Convert.ToInt32(ConfigData.HeaderDNT.Value).ToString());
emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", Convert.ToInt32(ConfigData.HeaderUpgradeInsecure.Value).ToString());
emulatedBrowserHeaders.Add("User-Agent", ConfigData.HeaderUserAgent.Value);
}
// Build WebRequest for index
var myIndexRequest = new WebRequest()
{
Type = RequestType.GET,
Url = SiteLink,
Headers = emulatedBrowserHeaders
};
// Get index page for cookies
output("\nGetting index page (for cookies).. with " + SiteLink);
var indexPage = await webclient.GetString(myIndexRequest);
// Building login form data
var pairs = new Dictionary<string, string> {
{ "username", ConfigData.Username.Value },
{ "password", ConfigData.Password.Value }
};
// Build WebRequest for login
var myRequestLogin = new WebRequest()
{
Type = RequestType.GET,
Url = LoginUrl,
Headers = emulatedBrowserHeaders,
Cookies = indexPage.Cookies,
Referer = SiteLink
};
// Get login page -- (not used, but simulation needed by tracker security's checks)
latencyNow();
output("\nGetting login page (user simulation).. with " + LoginUrl);
var loginPage = await webclient.GetString(myRequestLogin);
// Build WebRequest for submitting authentification
var request = new WebRequest()
{
PostData = pairs,
Referer = LoginUrl,
Type = RequestType.POST,
Url = LoginCheckUrl,
Headers = emulatedBrowserHeaders,
Cookies = indexPage.Cookies,
};
// Perform loggin
latencyNow();
output("\nPerform loggin.. with " + LoginCheckUrl);
var response = await webclient.GetString(request);
// Test if we are logged in
await ConfigureIfOK(response.Cookies, !string.IsNullOrEmpty(response.Cookies) && !response.IsRedirect, () =>
{
// Default error message
string message = "Error during attempt !";
// Parse redirect header
string redirectTo = response.RedirectingTo;
// Analyzer error code
if(redirectTo.Contains("login.php?error=4"))
{
// Set message
message = "Wrong username or password !";
}
// Oops, unable to login
output("-> Login failed: " + message, "error");
throw new ExceptionWithConfigData("Login failed: " + message, configData);
});
output("\nCookies saved for future uses...");
ConfigData.CookieHeader.Value = indexPage.Cookies + " " + response.Cookies + " ts_username="******"\n-> Login Success\n");
return IndexerConfigurationStatus.RequiresTesting;
}