public void TestUrlModification()
{
IHttpUrl url;
// Test the odd preservation of the question mark
Assert.AreEqual("http://example.com/foo?", new HttpUrl("example.com", "/foo?").ToFull());
Assert.AreEqual("http://example.com/foo", new HttpUrl("example.com", "/foo?").WithoutQuery("blah").ToFull()); // dubious: could keep the question mark if there was no such query name
Assert.AreEqual("http://example.com/foo", new HttpUrl("example.com", "/foo").WithQuery("blah", "s").WithoutQuery("blah").ToFull());
Assert.AreEqual("http://example.com/foo", new HttpUrl("example.com", "/foo?").WithQuery("blah", "s").WithoutQuery("blah").ToFull());
// WithQuery Single + Blank URL
url = new HttpUrl("example.com", "/foo").WithQuery("blah", "thingy");
Assert.IsTrue(url.HasQuery);
Assert.AreEqual("http://example.com/foo?blah=thingy", url.ToFull());
url = new HttpUrl("example.com", "/foo").WithQuery("blah", (string) null);
Assert.IsFalse(url.HasQuery);
Assert.AreEqual("http://example.com/foo", url.ToFull());
// Query string key without value
url = new HttpUrl("localhost", "/blah/thingy?dl");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
url = new HttpUrl("localhost", "/blah/thingy?dl=");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
url = new HttpUrl("localhost", "/blah/thingy?dl&blah=5");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "blah" && kvp.Value == "5"));
url = new HttpUrl("localhost", "/blah/thingy?dl=&blah=5");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "blah" && kvp.Value == "5"));
url = new HttpUrl("localhost", "/blah/thingy?blah=5&dl");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "blah" && kvp.Value == "5"));
url = new HttpUrl("localhost", "/blah/thingy?blah=5&dl=");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "blah" && kvp.Value == "5"));
url = new HttpUrl("localhost", "/blah/thingy?blah=5&dl&");
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "dl" && kvp.Value == ""));
Assert.IsTrue(url.Query.Any(kvp => kvp.Key == "blah" && kvp.Value == "5"));
// WithQuery Single + one other argument
url = new HttpUrl("example.com", "/foo?q=s").WithQuery("blah", "thingy");
Assert.IsTrue(url.HasQuery);
Assert.AreEqual("http://example.com/foo?q=s&blah=thingy", url.ToFull());
url = new HttpUrl("example.com", "/foo?q=s").WithQuery("blah", (string) null);
Assert.IsTrue(url.HasQuery);
Assert.AreEqual("http://example.com/foo?q=s", url.ToFull());
// WithQuery Single + just the same argument
url = new HttpUrl("example.com", "/foo?blah=s").WithQuery("blah", "thingy");
Assert.IsTrue(url.HasQuery);
Assert.AreEqual("http://example.com/foo?blah=thingy", url.ToFull());
url = new HttpUrl("example.com", "/foo?blah=s").WithQuery("blah", (string) null);
Assert.IsFalse(url.HasQuery);
Assert.AreEqual("http://example.com/foo", url.ToFull());
}
public void TestSTS(string secretId, string secretKey, string policy)
{
string camHost = "sts.api.qcloud.com";
string camPath = "/v2/index.php";
string camMethod = "GET";
bool isHttps = true;
Dictionary <string, string> queryParameters = new Dictionary <string, string>();
queryParameters.Add("policy", policy);
queryParameters.Add("name", "brady");
queryParameters.Add("Action", "GetFederationToken");
queryParameters.Add("SecretId", secretId);
queryParameters.Add("Nonce", new Random().Next(1, int.MaxValue).ToString());
long time = TimeUtils.GetCurrentTime(TimeUnit.SECONDS);
queryParameters.Add("Timestamp", time.ToString());
queryParameters.Add("RequestClient", "net-sdk-v5");
queryParameters.Add("durationSeconds", 7200.ToString());
string plainText = MakeSignPlainText(queryParameters, camMethod, camHost, camPath);
string hamcSha1 = DigestUtils.GetHamcSha1ToBase64(plainText, Encoding.UTF8, secretKey, Encoding.UTF8);
queryParameters.Add("Signature", hamcSha1);
HttpUrl httpUrl = new HttpUrl();
httpUrl.Scheme = isHttps ? "https" : "http";
httpUrl.Host = camHost;
httpUrl.Path = camPath;
Dictionary <string, string> tmp = new Dictionary <string, string>(queryParameters.Count);
foreach (KeyValuePair <string, string> pair in queryParameters)
{
tmp.Add(pair.Key, URLEncodeUtils.Encode(pair.Value).Replace("%20", "+"));
}
queryParameters.Clear();
httpUrl.SetQueryParameters(tmp);
SetSTSRequest(camMethod, isHttps, httpUrl);
}
internal static string MakeUri(ParameterInfo parameter, HttpUrl baseUrl, OperationInfo <Verb> operation)
{
string result = baseUrl.ToString();
foreach (var argument in operation.Arguments)
{
var argumentSource = argument.Source;
if (!(argumentSource is FromUrlAttribute))
{
continue;
}
var url = ((FromUrlAttribute)argumentSource).UrlTemplate.Replace(FromUrlAttribute.Key, "/" + argument.Parameter.Name + "/").TrimStart('/');
result += (result.EndsWith("/") ? String.Empty : "/") + url;
if (argument.Parameter == parameter)
{
break;
}
}
return(result);
}
public IList <HttpUrl> Find(HttpUrl sourcehttpUrl, List <string> errorStrings)
{
var httpUrls = new List <HttpUrl>();
foreach (var filePath in this.dictionaryFilePaths)
{
var file = new StreamReader(filePath);
string relativeFilePath;
while ((relativeFilePath = file.ReadLine()) != null)
{
var httpUrl = UrlParser.GetHttpUrl(sourcehttpUrl, "/" + relativeFilePath);
var response = this.browser.LoadWebsite(httpUrl.FullUrl, errorStrings);
if (response.IsSuccessStatusCode)
{
httpUrls.Add(httpUrl);
}
}
}
return(httpUrls);
}
public HttpUrlView(HttpUrl httpUrl, DataSource dataSource)
{
this.dataSource = dataSource;
this.httpUrl = httpUrl;
InitializeComponent();
builder = new DetailPageDataBuilder();
builder.Build("监控名称", httpUrl.Description);
builder.Build("监控链接", httpUrl.Url);
builder.Build("请求方法", httpUrl.Method);
builder.Build("请求延时", httpUrl.Delay + " ms");
var noticeTargetTask = Common.GetNoticeTargetNamesAsync(dataSource, httpUrl.NoticeTarget);
var statusTask = Common.GetStatusAsync(delegate {
return(dataSource.GetUrlStatus());
}, httpUrl.ID);
builder.Build("通知人员", noticeTargetTask.Result);
builder.Build("状态", statusTask.Result);
builder.Build("访问源", dataSource.ListenServerItem.Host);
ContentList.DataContext = builder.DataSource;
}
private static IEnumerable <Cookie> DecodeHeader(HttpUrl httpUrl, string header)
{
var result = new List <Cookie>();
var limit = header.Length;
int pairEnd;
for (var pos = 0; pos < limit; pos = pairEnd + 1)
{
pairEnd = DelimiterOffset(header, pos, limit, ";,");
var equalsSign = DelimiterOffset(header, pos, pairEnd, "=");
var name = TrimSubstring(header, pos, equalsSign);
if (name.StartsWith("$"))
{
continue;
}
// We have either name=value or just a name.
var value = equalsSign < pairEnd?TrimSubstring(header, equalsSign + 1, pairEnd) : "";
result.Add(new Cookie.Builder().Name(name).Value(value).Domain(httpUrl.Host()).Build());
}
return(result);
}
private static void CreateProxy(string targetDirectory, HttpUrl url)
{
var namedGraphSelector = _container.Resolve <INamedGraphSelector>() as ILocallyControlledNamedGraphSelector;
if (namedGraphSelector != null)
{
namedGraphSelector.NamedGraph = (Uri)url;
}
var apiDocumentation = GetApiDocumentation(url);
if (apiDocumentation == null)
{
throw new InvalidOperationException(String.Format("No API documentation found at '{0}'.", url));
}
if (apiDocumentation.SupportedClasses.Count == 0)
{
throw new InvalidOperationException(String.Format("No supported classes found at '{0}'.", url));
}
var generator = _container.Resolve <IClassGenerator>();
var supportedClasses = from supportedClass in apiDocumentation.SupportedClasses
where supportedClass.Id.ToString() != EntityConverter.Hydra + "ApiDocumentation"
from @class in generator.CreateCode(supportedClass)
select @class;
foreach (var @class in supportedClasses)
{
File.WriteAllText(Path.Combine(targetDirectory, @class.Key), @class.Value);
}
if (namedGraphSelector != null)
{
namedGraphSelector.NamedGraph = null;
}
}
public override string GetVideoUrl(VideoInfo video)
{
string cid = "cid=" + video.Other.ToString();
var p1 = GetWebData <JObject>(video.VideoUrl, cid);
CookieContainer cc = new CookieContainer();
string postData = "url=" + HttpUtility.UrlEncode(p1.Value <string>("url"));
string webData = GetWebData(@"http://www.telewizjada.net/set_cookie.php", postData, cc);
var jData = GetWebData <JObject>(@"http://www.telewizjada.net/get_channel_url.php", cid, cc);
var webData2 = jData.Value <string>("url");
string m3u = GetWebData(webData2, null, cc);
Match m = Regex.Match(m3u, @"(?<ch>chu.*)");
if (m.Success)
{
HttpUrl httpUrl = new HttpUrl(FormatDecodeAbsolutifyUrl(webData2, m.Groups["ch"].Value, "{0}", UrlDecoding.None));
httpUrl.LiveStream = true;
foreach (Cookie cookie in cc.GetCookies(new Uri(@"http://www.telewizjada.net")))
{
httpUrl.Cookies.Add(cookie);
}
return(httpUrl.ToString());
}
return(String.Empty);
}
public override string GetVideoUrl(string url)
{
string webData = WebCache.Instance.GetWebData(url);
webData = GetFromPost(url, webData);
if (!string.IsNullOrEmpty(webData))
{
string packed = null;
int i = webData.LastIndexOf(@"return p}");
if (i >= 0)
{
int j = webData.IndexOf(@"</script>", i);
if (j >= 0)
{
packed = webData.Substring(i + 9, j - i - 9);
}
}
if (!String.IsNullOrEmpty(packed))
{
packed = packed.Replace(@"\'", @"'");
string unpacked = Helpers.StringUtils.UnPack(packed);
return(Helpers.StringUtils.GetSubString(unpacked, @"name=""src""value=""", @""""));
}
return(String.Empty);
}
Match m = Regex.Match(webData, @"var\slnk1\s=\s'(?<url>[^']*)'");
if (m.Success)
{
HttpUrl httpUrl = new HttpUrl(m.Groups["url"].Value);
httpUrl.UserAgent = OnlineVideoSettings.Instance.UserAgent;
return(httpUrl.ToString());
}
return(String.Empty);
}
private void RecursiveScan(HttpUrl httpUrl, ScanResult scanResult)
{
this.RecursiveScan(httpUrl, new[] { httpUrl.Host }, scanResult);
}
public void Scan(HttpUrl httpUrl, IList <string> targetHosts)
{
var scanResult = new ScanResult();
this.RecursiveScan(httpUrl, targetHosts, scanResult);
}
public void Scan(HttpUrl httpUrl)
{
var scanResult = new ScanResult();
this.RecursiveScan(httpUrl, new[] { httpUrl.Host }, scanResult);
}
public void Teardown()
{
_url = null;
}
public void should_give_same_instance_if_no_fragments_are_given()
{
var expected = new HttpUrl(true, "http://temp.uri/", "http", "temp.uri", 80, "/", null, null);
expected.WithFragment(null).Should().BeSameAs(expected);
}
public override string GetVideoUrl(string url)
{
CookieContainer cc = new CookieContainer();
if (url.Contains("embed") || url.Contains(@"/f/"))
{
string page = WebCache.Instance.GetWebData(url);
if (!string.IsNullOrEmpty(page))
{
Match n = Regex.Match(page, @"file:\s'(?<url>[^']*)'");
if (n.Success)
{
HttpUrl httpurl = new HttpUrl(n.Groups["url"].Value);
httpurl.UserAgent = @"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko";
return(httpurl.ToString());
}
}
}
else
{
string page = WebCache.Instance.GetWebData(url, cookies: cc);
if (!string.IsNullOrEmpty(page))
{
string op = Regex.Match(page, @"<input\stype=""hidden""\sname=""op""\svalue=""(?<value>[^""]+)"">").Groups["value"].Value;
string id = Regex.Match(page, @"<input\stype=""hidden""\sname=""id""\svalue=""(?<value>[^""]+)"">").Groups["value"].Value;
string rand = Regex.Match(page, @"<input\stype=""hidden""\sname=""rand""\svalue=""(?<value>[^""]+)"">").Groups["value"].Value;
string referer = Regex.Match(page, @"<input\stype=""hidden""\sname=""referer""\svalue=""(?<value>[^""]+)"">").Groups["value"].Value;
string method_free = Regex.Match(page, @"<input\stype=""submit""\sname=""method_free""\svalue=""(?<value>[^""]+)"">").Groups["value"].Value;
//string method_premium = Regex.Match(page, @"<input\stype=""submit""\sname=""method_premium""\svalue=""(?<value>[^""]+)"">").Groups["value"].Value;
//string timeToWait = Regex.Match(page, @"<span\sid=""countdown_str"">[^>]*>[^>]*>[^>]*>(?<time>[^<]+)</span>").Groups["time"].Value;
//if (Convert.ToInt32(timeToWait) < 10)
{
string postdata = "op=" + op +
"&id=" + id +
"&rand=" + rand +
"&referer=" + referer +
"&method_free=" + HttpUtility.UrlEncode(method_free) +
//"&method_premium=" + method_premium +
"&down_direct=1";
//System.Threading.Thread.Sleep(Convert.ToInt32(timeToWait) * 1001);
string page2 = WebCache.Instance.GetWebData(url, postdata, cc, url);
if (!string.IsNullOrEmpty(page2))
{
string packed = null;
int i = page2.LastIndexOf(@"return p}");
if (i >= 0)
{
int j = page2.IndexOf(@"</script>", i);
if (j >= 0)
{
packed = page2.Substring(i + 9, j - i - 9);
}
}
string resUrl;
if (!String.IsNullOrEmpty(packed))
{
packed = packed.Replace(@"\'", @"'");
string unpacked = Helpers.StringUtils.UnPack(packed);
string res = Helpers.StringUtils.GetSubString(unpacked, @"'file','", @"'");
if (!String.IsNullOrEmpty(res))
{
resUrl = res;
}
else
{
resUrl = Helpers.StringUtils.GetSubString(unpacked, @"name=""src""value=""", @"""");
}
}
else
{
resUrl = Helpers.StringUtils.GetSubString(page2, @"addVariable('file','", @"'");
}
resUrl = resUrl.Replace("[", "%5b").Replace("]", "%5d");
return(resUrl);
}
}
}
}
return(String.Empty);
}
internal void Apply(HttpUrl httpUrl)
{
base.Apply(httpUrl);
httpUrl.OpenConnectionSleepTime = this.OpenConnectionSleepTime;
httpUrl.OpenConnectionTimeout = this.OpenConnectionTimeout;
httpUrl.TotalReopenConnectionTimeout = this.TotalReopenConnectionTimeout;
httpUrl.ServerAuthenticate = this.ServerAuthenticate;
httpUrl.ServerUserName = this.ServerUserName;
httpUrl.ServerPassword = this.ServerPassword;
httpUrl.ProxyServerAuthenticate = this.ProxyServerAuthenticate;
httpUrl.ProxyServer = this.ProxyServer;
httpUrl.ProxyServerPort = this.ProxyServerPort;
httpUrl.ProxyServerUserName = this.ProxyServerUserName;
httpUrl.ProxyServerPassword = this.ProxyServerPassword;
httpUrl.ProxyServerType = this.ProxyServerType;
}
internal void Apply(HttpUrl httpUrl)
{
base.Apply(httpUrl);
httpUrl.OpenConnectionSleepTime = OpenConnectionSleepTime;
httpUrl.OpenConnectionTimeout = OpenConnectionTimeout;
httpUrl.TotalReopenConnectionTimeout = TotalReopenConnectionTimeout;
}
public WebScanItemDisplay(HttpUrl httpUrl)
{
this.HttpUrl = httpUrl;
}
public PersonClient(HttpUrl baseUri, string authenticationScheme) : base(baseUri, authenticationScheme)
{
}
public ProductClient(HttpUrl baseUri) : base(baseUri)
{
}
public TypeClient(HttpUrl baseUri) : base(baseUri)
{
}
private void RecursiveScan(HttpUrl httpUrl, IList <string> targetHosts, ScanResult scanResult)
{
if (targetHosts.Any(x => httpUrl.Host.EndsWith(x)) == false)
{
return;
}
lock (this.httpUrlLock)
{
var storedHttpUrl = scanResult.GetHttpUrls().SingleOrDefault(x => x.Equals(httpUrl));
if (storedHttpUrl != null)
{
this.AddQueryString(httpUrl, storedHttpUrl.QueryString);
return;
}
scanResult.AddHttpUrl(httpUrl);
}
Console.WriteLine($"Scanning URL {httpUrl.FullUrl}");
var node = this.GetDocumentNode(httpUrl);
if (node.OuterLength <= 0)
{
return;
}
var content = node.OuterHtml;
var hrefs = node.GetHrefs();
scanResult.AddComments(httpUrl, node.GetComments());
Parallel.ForEach(hrefs, href =>
{
var hrefType = UrlParser.GetHrefType(href);
switch (hrefType)
{
case HrefType.Anchor:
break;
case HrefType.BrowserUrl:
scanResult.AddBrowserUrl(httpUrl, href);
break;
case HrefType.DataUrl:
scanResult.AddDataUrl(httpUrl, href);
break;
case HrefType.FtpUrl:
var host = UrlParser.GetHostFromUrl(href);
var path = UrlParser.GetPathFromUrl(href);
var fileName = UrlParser.GetFileNameFromUrl(href);
scanResult.AddFtpUrl(httpUrl, new FtpUrl(host, path, fileName));
break;
case HrefType.Javascript:
scanResult.AddJavascriptUrl(httpUrl, href);
break;
case HrefType.MailAddress:
scanResult.AddMailAddress(httpUrl, href);
break;
case HrefType.UnknownUrl:
scanResult.AddUnkownUrl(httpUrl, href);
break;
}
if (hrefType == HrefType.FullUrl)
{
var url = UrlParser.GetHttpUrl(href);
this.RecursiveScan(url, targetHosts, scanResult);
}
else if (hrefType == HrefType.RelativePath)
{
var url = UrlParser.GetHttpUrl(httpUrl, href);
this.RecursiveScan(url, targetHosts, scanResult);
}
});
}
private void AddWebScan(HttpUrl suggestedHttpUrl)
{
var selectedTargetDisplay = this.SelectedTargetDisplay;
if (suggestedHttpUrl == null)
{
suggestedHttpUrl = HttpUrl.Default;
}
var isAcknowledged = false;
var isInvalidUrl = false;
var webScanItemDisplay = new WebScanItemDisplay(suggestedHttpUrl);
do
{
var webScanDialog = new WebScanDialog(suggestedHttpUrl);
var webScanDialogResult = webScanDialog.ShowDialog();
var domain = webScanDialog.DomainText;
var port = webScanDialog.PortText;
var useHttps = webScanDialog.UseHttpsCheck.HasValue && webScanDialog.UseHttpsCheck.Value;
if (webScanDialogResult.HasValue && webScanDialogResult.Value)
{
var host = port == string.Empty ? domain : $"{domain}:{port}";
var httpUrl = new HttpUrl(host, string.Empty, string.Empty, useHttps);
webScanItemDisplay = new WebScanItemDisplay(httpUrl);
if (selectedTargetDisplay.WebScanItemDisplays.Any(x => x.HttpUrl.FullUrl == httpUrl.FullUrl))
{
return;
}
if (this.browser.LoadHeader(httpUrl.FullUrl).IsSuccessStatusCode)
{
foreach (var errorContentString in this.HandleHttpHeaderCompliance(webScanItemDisplay.HttpUrl))
{
webScanItemDisplay.ErrorContentStrings.Add(errorContentString);
}
isInvalidUrl = false;
}
else
{
isInvalidUrl = true;
}
}
isAcknowledged = webScanDialogResult.HasValue && webScanDialogResult.Value;
} while (isAcknowledged && isInvalidUrl);
if (isAcknowledged)
{
this.RunTask((uiThread) =>
{
uiThread.Execute(() =>
{
selectedTargetDisplay.WebScanItemDisplays.Add(webScanItemDisplay);
selectedTargetDisplay.SelectedWebScanItemDisplay = webScanItemDisplay;
});
});
}
}
public RobotsResult(RobotsType type, HttpUrl httpUrl)
{
this.Type = type;
this.HttpUrl = httpUrl;
}
public void TestNestedResolve()
{
var url = new HttpUrl();
url.SetHost("www.example.com");
url.SetLocation("/docgen/member/Color/ToString?thingy=stuff");
url.AssertComplete();
Assert.AreEqual(0, url.ParentDomains.Length);
Assert.AreEqual("www.example.com", url.Domain);
Assert.AreEqual("/docgen/member/Color/ToString", url.Path);
Assert.IsTrue(url.ParentPaths.SequenceEqual(new string[0]));
bool okProp = false, okDocGen = false;
var resolverDocGen = new UrlResolver(
new UrlMapping(path: "/member", handler: req =>
{
Assert.AreEqual("/Color/ToString", req.Url.Path);
Assert.AreEqual("/docgen/member/blah/xyz?thingy=stuff", req.Url.WithPath("/blah/xyz").ToHref());
var url2 = req.Url.WithPathOnly("/blah/xyz");
Assert.AreEqual("/docgen/member/blah/xyz", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
Assert.AreEqual("http://www.example.com/docgen/member/blah/xyz", url2.ToFull());
url2 = req.Url.WithPath("/blah/xyz");
Assert.AreEqual("/docgen/member/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
url2 = req.Url.WithPathParent().WithPath("/blah/xyz");
Assert.AreEqual("/docgen/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
url2 = req.Url.WithPathParent().WithPathParent().WithPath("/blah/xyz");
Assert.AreEqual("/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
okDocGen = true;
return(HttpResponse.Empty());
})
);
var resolverPropeller = new UrlResolver(
new UrlMapping(path: "/docgen", handler: req =>
{
Assert.AreEqual("/member/Color/ToString", req.Url.Path);
Assert.AreEqual("/docgen/blah/xyz?thingy=stuff", req.Url.WithPath("/blah/xyz").ToHref());
var url2 = req.Url.WithPathOnly("/blah/xyz");
Assert.AreEqual("/docgen/blah/xyz", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
Assert.AreEqual("http://www.example.com/docgen/blah/xyz", url2.ToFull());
url2 = req.Url.WithPath("/blah/xyz");
Assert.AreEqual("/docgen/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
url2 = req.Url.WithPathParent().WithPath("/blah/xyz");
Assert.AreEqual("/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
okProp = true;
return(resolverDocGen.Handle(req));
})
);
resolverPropeller.Handle(new HttpRequest()
{
Url = url
});
Assert.IsTrue(okProp);
Assert.IsTrue(okDocGen);
}
public PersonClient(HttpUrl baseUri) : base(baseUri)
{
}
private object TransformCollection(object result, HttpUrl requestUri, int totalItems, int skip, int take)
{
var entityContext = _entityContextProvider.EntityContext;
var collection = entityContext.Load<ICollection>((Uri)requestUri);
collection.TotalItems = totalItems;
collection.Members.Clear();
foreach (IEntity entity in (IEnumerable)result)
{
collection.Members.Add(entity.AsEntity<IResource>());
}
if ((skip <= 0) && (take <= 0))
{
entityContext.Commit();
return result;
}
var viewId = collection.Id.Uri.AddFragment("view");
var view = entityContext.Load<IPartialCollectionView>(viewId);
collection.View = view;
view.ItemsPerPage = (take > 0 ? take : totalItems);
entityContext.Commit();
return result;
}
public override string GetVideoUrl(VideoInfo video)
{
string resultUrl = GetFormattedVideoUrl(video);
string playListUrl = GetPlaylistUrl(resultUrl);
if (String.IsNullOrEmpty(playListUrl))
{
return(String.Empty);
}
string data = GetWebData(playListUrl);
Match m = Regex.Match(data, @"defaultQuality: ""(?<default>[^""]+)"",");
string defaultQuality = null;
if (m.Success)
{
defaultQuality = m.Groups["default"].Value;
}
m = Regex.Match(data, @"qualities:\s\[(?<qualities>[^]]+)]");
var baseOptions = base.GetPlaybackOptions(playListUrl);
string bareUrl = baseOptions.Values.First();
int inspos = bareUrl.IndexOf(".mp4");
video.PlaybackOptions = new Dictionary <string, string>();
if (m.Success)
{
string[] qualities = m.Groups["qualities"].Value.Split(',');
foreach (string quality in qualities)
{
var q = quality.Trim(new[] { '"', ' ' });
if (q == defaultQuality)
{
HttpUrl final = new HttpUrl(bareUrl);
final.Referer = playListUrl;
video.PlaybackOptions.Add(q, final.ToString());
}
else
if (!String.IsNullOrEmpty(q))
{
HttpUrl final = new HttpUrl(bareUrl.Insert(inspos, '-' + q));
final.Referer = playListUrl;
video.PlaybackOptions.Add(q, final.ToString());
}
}
}
if (video.PlaybackOptions.Count == 0)
{
video.PlaybackOptions = null;
}
m = Regex.Match(data, @"kind:\s""subtitles"",\ssrclang:\s""(?<langcode>[^""]+)"",\slabel:\s""(?<langname>[^""]*)"",\ssrc:\s""(?<url>[^""]+)""\s}");
Dictionary <string, string> subs = new Dictionary <string, string>();
while (m.Success)
{
subs.Add(m.Groups["langcode"].Value, m.Groups["url"].Value);
m = m.NextMatch();
}
string[] prefSubs = nativeSubtitleLanguages.Split(';');
foreach (string prefSub in prefSubs)
{
if (subs.ContainsKey(prefSub) && !String.IsNullOrEmpty(subs[prefSub]))
{
string subData = GetWebData(subs[prefSub]);
if (subData.StartsWith(@"WEBVTT"))
{
subData = Helpers.SubtitleUtils.Webvtt2SRT(subData);
}
video.SubtitleText = subData;
break;
}
}
if (String.IsNullOrEmpty(video.SubtitleUrl))
{
sh.SetSubtitleText(video, this.GetTrackingInfo);
}
if (video.PlaybackOptions == null)
{
HttpUrl final = new HttpUrl(bareUrl);
final.Referer = playListUrl;
return(final.ToString());
}
return(video.PlaybackOptions.Values.Last());
}
public void TestSkippableHandlers()
{
var url = new HttpUrl();
url.SetHost("www.example.com");
url.SetLocation("/docgen/member/Color/ToString?thingy=stuff");
url.AssertComplete();
Assert.AreEqual(0, url.ParentDomains.Length);
Assert.AreEqual("www.example.com", url.Domain);
var resolver = new UrlResolver(
new UrlMapping(req =>
{
Assert.AreEqual(1, req.Url.ParentDomains.Length);
Assert.AreEqual("www.example.com", req.Url.ParentDomains[0]);
Assert.AreEqual("", req.Url.Domain);
return null;
}, "www.example.com", skippable: true),
new UrlMapping(req =>
{
Assert.AreEqual(1, req.Url.ParentDomains.Length);
Assert.AreEqual("example.com", req.Url.ParentDomains[0]);
Assert.AreEqual("www.", req.Url.Domain);
return null;
}, "example.com", skippable: true),
new UrlMapping(req =>
{
Assert.AreEqual(0, req.Url.ParentDomains.Length);
Assert.AreEqual("www.example.com", req.Url.Domain);
return null;
}, skippable: true),
new UrlMapping(req =>
{
Assert.AreEqual(0, req.Url.ParentDomains.Length);
Assert.AreEqual("www.example.com", req.Url.Domain);
return HttpResponse.PlainText("blah");
}
));
resolver.Handle(new HttpRequest { Url = url });
}
/// <summary>Exposes the <see cref="System.Security.Policy.Url" /> assertions extension.</summary>
/// <param name="url">The subject URL.</param>
/// <returns>Assertions for a given <paramref name="url" />.</returns>
public static HttpUrlAssertions Should(this HttpUrl url)
{
return(new HttpUrlAssertions(url));
}
public void TestNestedResolve()
{
var url = new HttpUrl();
url.SetHost("www.example.com");
url.SetLocation("/docgen/member/Color/ToString?thingy=stuff");
url.AssertComplete();
Assert.AreEqual(0, url.ParentDomains.Length);
Assert.AreEqual("www.example.com", url.Domain);
Assert.AreEqual("/docgen/member/Color/ToString", url.Path);
Assert.IsTrue(url.ParentPaths.SequenceEqual(new string[0]));
bool okProp = false, okDocGen = false;
var resolverDocGen = new UrlResolver(
new UrlMapping(path: "/member", handler: req =>
{
Assert.AreEqual("/Color/ToString", req.Url.Path);
Assert.AreEqual("/docgen/member/blah/xyz?thingy=stuff", req.Url.WithPath("/blah/xyz").ToHref());
var url2 = req.Url.WithPathOnly("/blah/xyz");
Assert.AreEqual("/docgen/member/blah/xyz", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
Assert.AreEqual("http://www.example.com/docgen/member/blah/xyz", url2.ToFull());
url2 = req.Url.WithPath("/blah/xyz");
Assert.AreEqual("/docgen/member/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
url2 = req.Url.WithPathParent().WithPath("/blah/xyz");
Assert.AreEqual("/docgen/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
url2 = req.Url.WithPathParent().WithPathParent().WithPath("/blah/xyz");
Assert.AreEqual("/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
okDocGen = true;
return HttpResponse.Empty();
})
);
var resolverPropeller = new UrlResolver(
new UrlMapping(path: "/docgen", handler: req =>
{
Assert.AreEqual("/member/Color/ToString", req.Url.Path);
Assert.AreEqual("/docgen/blah/xyz?thingy=stuff", req.Url.WithPath("/blah/xyz").ToHref());
var url2 = req.Url.WithPathOnly("/blah/xyz");
Assert.AreEqual("/docgen/blah/xyz", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
Assert.AreEqual("http://www.example.com/docgen/blah/xyz", url2.ToFull());
url2 = req.Url.WithPath("/blah/xyz");
Assert.AreEqual("/docgen/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
url2 = req.Url.WithPathParent().WithPath("/blah/xyz");
Assert.AreEqual("/blah/xyz?thingy=stuff", url2.ToHref());
Assert.AreEqual("/blah/xyz", url2.Path);
okProp = true;
return resolverDocGen.Handle(req);
})
);
resolverPropeller.Handle(new HttpRequest() { Url = url });
Assert.IsTrue(okProp);
Assert.IsTrue(okDocGen);
}
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var java_uri = request.RequestUri.GetComponents(UriComponents.AbsoluteUri, UriFormat.UriEscaped);
var url = new Java.Net.URL(java_uri);
var body = default(RequestBody);
if (request.Content != null)
{
var bytes = await request.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
var contentType = "text/plain";
if (request.Content.Headers.ContentType != null)
{
contentType = string.Join(" ", request.Content.Headers.GetValues("Content-Type"));
}
body = RequestBody.Create(MediaType.Parse(contentType), bytes);
}
var requestBuilder = new Request.Builder()
.Method(request.Method.Method.ToUpperInvariant(), body)
.Url(url);
if (DisableCaching)
{
requestBuilder.CacheControl(noCacheCacheControl);
}
var keyValuePairs = request.Headers
.Union(request.Content != null ?
request.Content.Headers :
Enumerable.Empty <KeyValuePair <string, IEnumerable <string> > >());
// Add Cookie Header if there's any cookie for the domain in the cookie jar
var stringBuilder = new StringBuilder();
if (client.CookieJar() != null)
{
var jar = client.CookieJar();
var cookies = jar.LoadForRequest(HttpUrl.Get(url));
foreach (var cookie in cookies)
{
stringBuilder.Append(cookie.Name() + "=" + cookie.Value() + ";");
}
}
foreach (var kvp in keyValuePairs)
{
if (kvp.Key == "Cookie")
{
foreach (var val in kvp.Value)
{
stringBuilder.Append(val + ";");
}
}
else
{
requestBuilder.AddHeader(kvp.Key, String.Join(getHeaderSeparator(kvp.Key), kvp.Value));
}
}
if (stringBuilder.Length > 0)
{
requestBuilder.AddHeader("Cookie", stringBuilder.ToString().TrimEnd(';'));
}
if (Timeout != null)
{
var clientBuilder = client.NewBuilder();
var timeout = (long)Timeout.Value.TotalSeconds;
clientBuilder.ConnectTimeout(timeout, TimeUnit.Seconds);
clientBuilder.WriteTimeout(timeout, TimeUnit.Seconds);
clientBuilder.ReadTimeout(timeout, TimeUnit.Seconds);
client = clientBuilder.Build();
}
cancellationToken.ThrowIfCancellationRequested();
var rq = requestBuilder.Build();
var call = client.NewCall(rq);
// NB: Even closing a socket must be done off the UI thread. Cray!
cancellationToken.Register(() => Task.Run(() => call.Cancel()));
var resp = default(Response);
try
{
resp = await call.EnqueueAsync().ConfigureAwait(false);
var newReq = resp.Request();
var newUri = newReq == null ? null : newReq.Url().Uri();
request.RequestUri = new Uri(newUri.ToString());
if (throwOnCaptiveNetwork && newUri != null)
{
if (url.Host != newUri.Host)
{
throw new CaptiveNetworkException(new Uri(java_uri), new Uri(newUri.ToString()));
}
}
}
catch (IOException ex)
{
if (ex.Message.ToLowerInvariant().Contains("canceled"))
{
throw new System.OperationCanceledException();
}
// Calling HttpClient methods should throw .Net Exception when fail #5
throw new HttpRequestException(ex.Message, ex);
}
var respBody = resp.Body();
cancellationToken.ThrowIfCancellationRequested();
var ret = new HttpResponseMessage((HttpStatusCode)resp.Code());
ret.RequestMessage = request;
ret.ReasonPhrase = resp.Message();
// ReasonPhrase is empty under HTTPS #8
if (string.IsNullOrEmpty(ret.ReasonPhrase))
{
try
{
ret.ReasonPhrase = ((ReasonPhrases)resp.Code()).ToString().Replace('_', ' ');
}
#pragma warning disable 0168
catch (Exception ex)
{
ret.ReasonPhrase = "Unassigned";
}
#pragma warning restore 0168
}
if (respBody != null)
{
var content = new ProgressStreamContent(respBody.ByteStream(), CancellationToken.None);
content.Progress = getAndRemoveCallbackFromRegister(request);
ret.Content = content;
}
else
{
ret.Content = new ByteArrayContent(new byte[0]);
}
var respHeaders = resp.Headers();
foreach (var k in respHeaders.Names())
{
ret.Headers.TryAddWithoutValidation(k, respHeaders.Get(k));
ret.Content.Headers.TryAddWithoutValidation(k, respHeaders.Get(k));
}
return(ret);
}
//Constructor
public HttpRequest(HttpListenerContext context)
{
Context = context;
Url = new HttpUrl(this);
}
private void CopyLink(HttpUrl httpUrl)
{
Clipboard.SetText(httpUrl.FullUrl);
}
public override string GetVideoUrl(VideoInfo video)
{
if ("livestream".Equals(video.Other))
{
string data = GetWebData(video.VideoUrl);
Match m2 = Regex.Match(data, @"<param\sname=""flashvars""\svalue=""assetID=(?<assetid>[^_]*)_");
if (m2.Success)
{
data = GetWebData(String.Format(@"http://www.rtve.es/api/videos/{0}/config/portada.json", m2.Groups["assetid"].Value));
m2 = Regex.Match(data, @"""file"":""(?<url>[^""]*)""");
if (m2.Success)
{
RtmpUrl rtmpUrl = new RtmpUrl(m2.Groups["url"].Value)
{
Live = true,
SwfUrl = @"http://www.rtve.es/swf/4.0.32/RTVEPlayerVideo.swf"
};
return(rtmpUrl.ToString());
}
}
return(null);
}
//source: http://code.google.com/p/xbmc-tvalacarta/source/browse/trunk/tvalacarta/servers/rtve.py:
// thanks to aabilio and tvalacarta
string[] parts = video.VideoUrl.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
string data2 = GetWebData(String.Format(@"http://www.rtve.es/ztnr/movil/thumbnail/anubis/videos/{0}.png", parts[parts.Length - 1]), referer: @"http://www.rtve.es");
data2 = Encoding.ASCII.GetString(Convert.FromBase64String(data2));
Match mm = Regex.Match(data2, ".*tEXt(?<cypher>.*)#[\x00]*(?<key>[0-9]*).*");
if (mm.Success)
{
string cypher = mm.Groups["cypher"].Value;
string key = mm.Groups["key"].Value;
string int_cypher = "";
int inc = 1;
int ti = 0;
while (ti < cypher.Length)
{
ti = ti + inc;
if (ti > 0 && ti <= cypher.Length)
{
int_cypher += cypher[ti - 1];
}
inc++;
if (inc == 5)
{
inc = 1;
}
}
string plaintext = "";
int key_ind = 0;
inc = 4;
while (key_ind < key.Length)
{
key_ind++;
ti = ((byte)key[key_ind - 1] - 48) * 10;
key_ind += inc;
if (key_ind <= key.Length)
{
ti += (byte)key[key_ind - 1] - 48;
}
ti++;
inc++;
if (inc == 5)
{
inc = 1;
}
if (ti > 0 && ti <= int_cypher.Length)
{
plaintext += int_cypher[ti - 1];
}
}
int p = plaintext.IndexOf("/resources");
string url = @"http://flash.akamaihd.multimedia.cdn.rtve.es/auth" + plaintext.Substring(p) +
"?v=2.6.8&fp=WIN%2016,0,0,305&r=TDBDO&g=UZEYDOLYKFLY";
string data = GetWebData(url);
p = data.IndexOf('?');
string finalUrl = plaintext.Substring(0, p) + data.Substring(0, p);
finalUrl = Regex.Replace(finalUrl, @"(//.*?.rtve.es)", @"//mvod.lvlt.rtve.es");
HttpUrl res = new HttpUrl(finalUrl);
res.Cookies.Add(new Cookie("odin", "odin=banebdyede"));
return(res.ToString());
}
return(null);
}
