/// <summary>Append a relative path to a Uri, handling traling slashes appropiately.</summary>
/// <param name="uri">The base Uri. </param>
/// <param name="relativeOrAbslouteUri">The relative or absloute URI. </param>
/// <param name="sep">The seperator. </param>
/// <returns>The appended Uri. </returns>
internal static Uri AppendPathToUri(Uri uri, string relativeOrAbslouteUri, string sep)
{
Uri relativeUri;
// Because of URI's Scheme, URI.TryCreate() can't differentiate a string with colon from an absolute URI.
// A workaround is added here to verify if a given string is an absolute URI.
if (Uri.TryCreate(relativeOrAbslouteUri, UriKind.Absolute, out relativeUri)
&& (relativeUri.Scheme == "http" || relativeUri.Scheme == "https"))
{
// Handle case if relPath is an absolute Uri
if (uri.IsBaseOf(relativeUri))
{
return relativeUri;
}
// Happens when using fiddler, DNS aliases, or potentially NATs
var absoluteUri = new Uri(relativeOrAbslouteUri);
return new Uri(uri, absoluteUri.AbsolutePath);
}
var ub = new UriBuilder(uri);
var appendString = ub.Path.EndsWith(sep) ? relativeOrAbslouteUri : sep + relativeOrAbslouteUri;
var escapedRelativeOrAbslouteUri = Uri.EscapeUriString(appendString);
ub.Path += escapedRelativeOrAbslouteUri;
return ub.Uri;
}
public void CreateRequestUsesBaseUri()
{
var baseUri = new Uri("http://www.example.com");
var factory = new RequestFactory(baseUri, "apiKey");
var request = factory.CreateRequest("test");
Assert.IsTrue(baseUri.IsBaseOf(request.RequestUri), "Request does not have expected base URI.");
}
public string ToRemote(string localPath)
{
foreach (var mapping in localRemote) {
var localBase = new Uri(mapping.Item1, UriKind.RelativeOrAbsolute);
var localUrl = new Uri(localPath, UriKind.RelativeOrAbsolute);
if (localBase.IsBaseOf(localUrl))
return mapping.Item2 + localBase.MakeRelativeUri(localUrl).OriginalString;
}
return null;
}
internal static Uri GetBaseUriToWrite(Uri rootBase, Uri currentBase)
{
if ((rootBase == currentBase) || (currentBase == null))
{
return null;
}
if ((rootBase != null) && ((rootBase.IsAbsoluteUri && currentBase.IsAbsoluteUri) && rootBase.IsBaseOf(currentBase)))
{
return rootBase.MakeRelativeUri(currentBase);
}
return currentBase;
}
/// <summary>
/// Reads a token from the current user's Visual Studio hive in the Windows Registry.
/// </summary>
/// <param name="targetUri">Key used to select the token.</param>
/// <param name="token">If successful, the token from the registry; otherwise `null`.</param>
/// <returns>True if successful; otherwise false.</returns>
public bool ReadToken(TargetUri targetUri, out Token token)
{
BaseSecureStore.ValidateTargetUri(targetUri);
Trace.WriteLine("TokenRegistry::ReadToken");
foreach (var key in EnumerateKeys(false))
{
if (key == null)
continue;
string url;
string type;
string value;
if (KeyIsValid(key, out url, out type, out value))
{
try
{
Uri tokenUri = new Uri(url);
if (tokenUri.IsBaseOf(targetUri.ActualUri))
{
byte[] data = Convert.FromBase64String(value);
data = ProtectedData.Unprotect(data, null, DataProtectionScope.CurrentUser);
value = Encoding.UTF8.GetString(data);
TokenType tokenType;
if (String.Equals(type, "Federated", StringComparison.OrdinalIgnoreCase))
{
tokenType = TokenType.Federated;
}
else
{
throw new InvalidOperationException("Unexpected token type encountered");
}
token = new Token(value, tokenType);
return true;
}
}
catch
{
Trace.WriteLine(" token read from registry was corrupt");
}
}
}
token = null;
return false;
}
public void Show(string account, Uri authenticationUri, Uri redirectUri)
{
if (window == null) {
waitHandle = new EventWaitHandle(false, EventResetMode.AutoReset);
var uiThread = new Thread(() => {
window = new Window() { Title = account };
window.Closing += (s, e) => {
window.Hide();
waitHandle.Set();
e.Cancel = true;
};
browser = new WebBrowser();
browser.Loaded += (s, e) => {
browser.Navigate(authenticationUri);
};
browser.Navigating += (s, e) => {
if (redirectUri.IsBaseOf(e.Uri) && redirectUri.AbsolutePath == e.Uri.AbsolutePath) {
var parameters = new NameValueCollection();
foreach (var parameter in e.Uri.Query.TrimStart('?').Split('&')) {
var nameValue = parameter.Split('=');
parameters.Add(nameValue[0], nameValue[1]);
}
var handler = Authenticated;
handler?.Invoke(this, new AuthenticatedEventArgs(parameters));
e.Cancel = true;
}
};
browser.Navigated += (s, e) => {
if (authenticationUri.IsBaseOf(e.Uri))
SetForegroundWindow(new WindowInteropHelper(window).Handle);
};
window.Content = browser;
window.Show();
System.Windows.Threading.Dispatcher.Run();
});
uiThread.SetApartmentState(ApartmentState.STA);
uiThread.Start();
} else {
window.Dispatcher.Invoke(() => {
browser.Source = authenticationUri;
window.Title = account;
window.Show();
});
}
waitHandle.WaitOne();
}
static public int IsBaseOf(IntPtr l)
{
try {
System.Uri self = (System.Uri)checkSelf(l);
System.Uri a1;
checkType(l, 2, out a1);
var ret = self.IsBaseOf(a1);
pushValue(l, true);
pushValue(l, ret);
return(2);
}
catch (Exception e) {
return(error(l, e));
}
}
static int IsBaseOf(IntPtr L)
{
try
{
ToLua.CheckArgsCount(L, 2);
System.Uri obj = (System.Uri)ToLua.CheckObject <System.Uri>(L, 1);
System.Uri arg0 = (System.Uri)ToLua.CheckObject <System.Uri>(L, 2);
bool o = obj.IsBaseOf(arg0);
LuaDLL.lua_pushboolean(L, o);
return(1);
}
catch (Exception e)
{
return(LuaDLL.toluaL_exception(L, e));
}
}
protected void SetUri(System.Uri baseAddress, string relativeAddress)
{
System.Uri uri = baseAddress;
if (relativeAddress != string.Empty)
{
if (!baseAddress.AbsolutePath.EndsWith("/", StringComparison.Ordinal))
{
UriBuilder uriBuilder = new UriBuilder(baseAddress);
Microsoft.ServiceBus.Channels.TransportChannelListener.FixIpv6Hostname(uriBuilder, baseAddress);
uriBuilder.Path = string.Concat(uriBuilder.Path, "/");
baseAddress = uriBuilder.Uri;
}
uri = new System.Uri(baseAddress, relativeAddress);
if (!baseAddress.IsBaseOf(uri))
{
baseAddress = uri;
}
}
if (!baseAddress.Scheme.Equals("sbwss"))
{
this.baseUri = baseAddress;
this.ValidateUri(uri);
this.uri = uri;
return;
}
UriBuilder uriBuilder1 = new UriBuilder(baseAddress)
{
Scheme = "sb",
Port = RelayEnvironment.RelayHttpsPort
};
UriBuilder uriBuilder2 = uriBuilder1;
this.baseUri = uriBuilder2.Uri;
UriBuilder uriBuilder3 = new UriBuilder(uri)
{
Scheme = "sb",
Port = RelayEnvironment.RelayHttpsPort
};
uriBuilder2 = uriBuilder3;
this.ValidateUri(uriBuilder2.Uri);
this.uri = uriBuilder2.Uri;
}
protected void SetUri(System.Uri baseAddress, string relativeAddress)
{
System.Uri uri = baseAddress;
if (relativeAddress != string.Empty)
{
if (!baseAddress.AbsolutePath.EndsWith("/", StringComparison.Ordinal))
{
UriBuilder uriBuilder = new UriBuilder(baseAddress);
TcpChannelListener.FixIpv6Hostname(uriBuilder, baseAddress);
uriBuilder.Path = uriBuilder.Path + "/";
baseAddress = uriBuilder.Uri;
}
uri = new System.Uri(baseAddress, relativeAddress);
if (!baseAddress.IsBaseOf(uri))
{
baseAddress = uri;
}
}
this.baseUri = baseAddress;
this.ValidateUri(uri);
this.uri = uri;
}
/// <summary>
/// Evaluates whether the specified path is beneath the specified root content directory, and if so, attempts to
/// determine the path of the project file which produced it.
/// </summary>
/// <param name="root">The root content directory.</param>
/// <param name="path">The path for which to find the original content path.</param>
/// <returns>The original content path, if it was found; otherwise, <paramref name="path"/>.</returns>
public static String GetOriginalContentFilePath(String root, String path)
{
Contract.Require(root, "root");
Contract.RequireNotEmpty(path, "path");
var entryAssembly = Assembly.GetEntryAssembly();
if (entryAssembly == null)
return path;
// Produce a URI which is relative to the directory that contains our application executable.
var uriExe = GetDirectoryUri(Path.GetDirectoryName(entryAssembly.Location));
var uriRoot = new Uri(uriExe, root);
var uriPathAbs = new Uri(Path.GetFullPath(path), UriKind.Absolute);
if (!uriRoot.IsBaseOf(uriPathAbs))
return path;
var uriPathRel = uriExe.MakeRelativeUri(uriPathAbs);
// Get out of "Debug" (or "Release", etc.)...
var dir = new DirectoryInfo(uriExe.LocalPath);
if (dir.Parent == null)
return path;
dir = dir.Parent;
// Get out of "bin"...
if (dir.Parent == null)
return path;
dir = dir.Parent;
// If we found a valid file, return its path, otherwise return the original path.
var uriOriginalContentDir = GetDirectoryUri(dir.FullName);
var uriOriginalContentFile = new Uri(uriOriginalContentDir, uriPathRel);
return File.Exists(uriOriginalContentFile.LocalPath) ? uriOriginalContentFile.LocalPath : path;
}
public bool IsEndpointFor(string other)
{
var baseAddress = this.Uri.ToString();
// HACK! To support Fiddler2 on Win8, localhost needs to be spelled out as localhost.fiddler, but still functions as localhost
baseAddress = baseAddress.Replace("localhost.fiddler", "localhost");
var baseUri = new Uri(delimit(baseAddress));
other = delimit(other.Replace("localhost.fiddler", "localhost"));
return baseUri.IsBaseOf(new Uri(other,UriKind.RelativeOrAbsolute));
}
public void Ldapx_Methods ()
{
Uri uri = new Uri ("ldapx://www.mono-project.com/");
Assert.AreEqual (String.Empty, uri.GetComponents (UriComponents.Path, UriFormat.SafeUnescaped), "GetComponents");
Assert.IsTrue (uri.IsBaseOf (uri), "IsBaseOf");
Assert.IsTrue (uri.IsWellFormedOriginalString (), "IsWellFormedOriginalString");
// ??? our parser doesn't seems to be called :(
}
public void NewsX_Methods ()
{
Uri uri = new Uri ("newsx://go-mono.com/");
Assert.AreEqual ("//go-mono.com/", uri.GetComponents (UriComponents.Path, UriFormat.SafeUnescaped), "GetComponents");
Assert.IsTrue (uri.IsBaseOf (uri), "IsBaseOf");
Assert.IsTrue (uri.IsWellFormedOriginalString (), "IsWellFormedOriginalString");
// ??? our parser doesn't seems to be called :(
}
void workerCrawler_DoWork(ref int progress,
ref string result, ref List<string> array, params object[] args)
{
array = new List<string>();
List<string> invalidLinks = new List<string>();
// Get the value which passed to this operation.
string urlToCheck = string.Empty;
Uri startingUri = new Uri(args[0].ToString());
result = "Crawling Links started.....";
Thread.Sleep(100);
var queue = new Queue<Uri>();
queue.Enqueue(startingUri);
array.Add(startingUri.ToString());
//get the base url
Uri uri = new Uri(startingUri.ToString());
string baseUrl = uri.GetLeftPart(UriPartial.Authority);
string hostName = uri.Host;
string Rstring;
#region queue
while (queue.Count > 0)
{
string sUrl = queue.Dequeue().ToString();
try
{
string html = new WebClient().DownloadString(sUrl);
//if html file is empty if (html.Length == 0) { continue; }
MatchCollection matches = Regex.Matches(html, "href[ ]*=[ ]*['|\"][^\"'\r\n]*['|\"]");
//MatchCollection matches = Regex.Matches(html, "(?<=<a\\s*?href=(?:'|\"))[^'\"]*?(?=(?:'|\"))");
foreach (Match match in matches)
{
string value = match.Value;
value = Regex.Replace(value, "(href[ ]*=[ ]*')|(href[ ]*=[ ]*\")", string.Empty);
if (value.EndsWith("\"") || value.EndsWith("'"))
value = value.Remove(value.Length - 1, 1);
if (!Regex.Match(value, @"\((.*)\)").Success)
{
try
{
Uri tempUrl = new Uri(value, UriKind.RelativeOrAbsolute);
if (startingUri.IsBaseOf(tempUrl)) //If URL of that site
{
if (!value.Contains("http:"))
{
Uri baseUri = new Uri(startingUri.ToString());
Uri absoluteUri = new Uri(baseUri, value);
value = absoluteUri.ToString();
}
//}
//Uri urlNext = new Uri(value, UriKind.RelativeOrAbsolute);
if (array.Contains(value)) continue;
if (invalidLinks.Contains(value)) continue;
//Discard following types of files
Regex invalidType = new Regex(@"(css|jpg|gif|pdf|doc|docx|ppt|pptx|js|png|ico|zip|xls|txt|exe)");
Match matchResult = invalidType.Match(value);
if (!matchResult.Success)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(value);
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.OK)
{
queue.Enqueue(new Uri(value));
array.Add(value);
result = "Link found: " + value;
Thread.Sleep(100);
urlNum++;
}
else
{
result = response.StatusDescription;
invalidLinks.Add(value);
}
}
Session["linksCrawled"] = array;
}
/*
var thread = new Thread(ProcessWebRequests);
threadList.Add(thread);
thread.Start();
*/
//ThreadFunction(value);
}
}
catch (Exception)//UriFormatException
{
result = "Invalid Uri";
}
}
}
}
catch
{
result = "Html Error";
//remove it from arrUrlStr if exist
//if (arrUrlStr.Contains(value)) remove state
///
/// If downloading times out, just ignore...
///
}
}
#endregion
progress = 100;
result = "Crawling done";
}
internal static bool IsBaseOf(Uri baseUriWithSlash, Uri requestUri)
{
return baseUriWithSlash.IsBaseOf(requestUri);
}
public static bool IsLink(this string url,
Uri baseUri, LinkKind kind, UriFilter filter = UriFilter.None)
{
Contract.Requires(baseUri.IsBaseUri());
if (string.IsNullOrWhiteSpace(url))
return false;
url = url.Trim().ToLower();
if (!Uri.IsWellFormedUriString(url, UriKind.RelativeOrAbsolute))
return false;
if (url.StartsWith("#"))
return false;
Uri uri;
if (!Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out uri))
return false;
if (!uri.IsAbsoluteUri)
{
if (!Uri.TryCreate(baseUri, uri, out uri))
return false;
}
if (uri.Scheme != "http")
return false;
switch (filter)
{
case UriFilter.LocalPath:
if (!baseUri.IsBaseOf(uri))
return false;
break;
case UriFilter.Authority:
if (uri.Authority != baseUri.Authority)
return false;
break;
}
if (kind == LinkKind.HTML)
return true;
if (!string.IsNullOrWhiteSpace(uri.Query))
return false;
string localPath;
try
{
localPath = Path.GetFileName(uri.LocalPath);
}
catch
{
return false;
}
switch (Path.GetExtension(localPath))
{
case ".jpg":
case ".jpeg":
case ".png":
case ".gif":
case ".bmp":
case ".tiff":
return true;
default:
return false;
}
}
public void IsBaseOf ()
{
Uri http = new Uri ("http://www.mono-project.com/Main_Page#FAQ?Edit");
Assert.IsTrue (http.IsBaseOf (http), "http-http");
Uri u = new Uri ("http://www.mono-project.com/Main_Page#FAQ");
Assert.IsTrue (u.IsBaseOf (http), "http-1a");
Assert.IsTrue (http.IsBaseOf (u), "http-1b");
u = new Uri ("http://www.mono-project.com/Main_Page");
Assert.IsTrue (u.IsBaseOf (http), "http-2a");
Assert.IsTrue (http.IsBaseOf (u), "http-2b");
u = new Uri ("http://www.mono-project.com/");
Assert.IsTrue (u.IsBaseOf (http), "http-3a");
Assert.IsTrue (http.IsBaseOf (u), "http-3b");
u = new Uri ("http://www.mono-project.com/Main_Page/");
Assert.IsFalse (u.IsBaseOf (http), "http-4a");
Assert.IsTrue (http.IsBaseOf (u), "http-4b");
// docs says the UserInfo isn't evaluated, but...
u = new Uri ("http://*****:*****@www.mono-project.com/Main_Page");
Assert.IsFalse (u.IsBaseOf (http), "http-5a");
Assert.IsFalse (http.IsBaseOf (u), "http-5b");
// scheme case sensitive ? no
u = new Uri ("HTTP://www.mono-project.com/Main_Page");
Assert.IsTrue (u.IsBaseOf (http), "http-6a");
Assert.IsTrue (http.IsBaseOf (u), "http-6b");
// host case sensitive ? no
u = new Uri ("http://www.Mono-Project.com/Main_Page");
Assert.IsTrue (u.IsBaseOf (http), "http-7a");
Assert.IsTrue (http.IsBaseOf (u), "http-7b");
// path case sensitive ? no
u = new Uri ("http://www.Mono-Project.com/MAIN_Page");
Assert.IsTrue (u.IsBaseOf (http), "http-8a");
Assert.IsTrue (http.IsBaseOf (u), "http-8b");
// different scheme
u = new Uri ("ftp://www.mono-project.com/Main_Page");
Assert.IsFalse (u.IsBaseOf (http), "http-9a");
Assert.IsFalse (http.IsBaseOf (u), "http-9b");
// different host
u = new Uri ("http://www.go-mono.com/Main_Page");
Assert.IsFalse (u.IsBaseOf (http), "http-10a");
Assert.IsFalse (http.IsBaseOf (u), "http-10b");
// different port
u = new Uri ("http://www.mono-project.com:8080/");
Assert.IsFalse (u.IsBaseOf (http), "http-11a");
Assert.IsFalse (http.IsBaseOf (u), "http-11b");
// specify default port
u = new Uri ("http://www.mono-project.com:80/");
Assert.IsTrue (u.IsBaseOf (http), "http-12a");
Assert.IsTrue (http.IsBaseOf (u), "http-12b");
}
private IEnumerable<KeyValuePair<string, string>> GetSourceFiles(string pdbFile, string toolPath)
{
var projectBaseUrl = new Uri (Path.GetFullPath (ProjectBaseDirectory + "\\"));
int exitCode;
string extractedCommandOutput = Execute (Path.Combine (toolPath, "srctool.exe"), "-r " + pdbFile, true, out exitCode);
using (var reader = new StringReader (extractedCommandOutput))
{
while (true)
{
var line = reader.ReadLine();
if (line == null)
yield break;
if (string.IsNullOrWhiteSpace (line))
continue;
// last line is srctool.exe summary.
Uri sourceFileUrl;
if (!Uri.TryCreate (line, UriKind.Absolute, out sourceFileUrl))
continue;
if (!projectBaseUrl.IsBaseOf (sourceFileUrl))
continue;
var relativeSourceFileUrl = projectBaseUrl.MakeRelativeUri (sourceFileUrl);
yield return new KeyValuePair<string, string> (line, relativeSourceFileUrl.ToString());
}
}
}
/// <summary>
/// Get every WebPage.Internal on a web site (or part of a web site) visiting all internal links just once
/// plus every external page (or other Url) linked to the web site as a WebPage.External
/// </summary>
/// <remarks>
/// Use .OfType WebPage.Internal to get just the internal ones if that's what you want
/// </remarks>
public static IEnumerable<WebPage> GetAllPagesUnder(Uri urlRoot)
{
int safetyCount = 0;
var queue = new Queue<Uri>();
var allSiteUrls = new HashSet<Uri>();
queue.Enqueue(urlRoot);
allSiteUrls.Add(urlRoot);
while (queue.Count > 0 && safetyCount++<5)
{
Uri url = queue.Dequeue();
HttpWebRequest oReq = (HttpWebRequest)WebRequest.Create(url);
oReq.UserAgent = @"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5";
HttpWebResponse resp = (HttpWebResponse)oReq.GetResponse();
WebPage result;
if (resp.ContentType.StartsWith("text/html", StringComparison.InvariantCultureIgnoreCase))
{
HtmlDocument doc = new HtmlDocument();
try
{
var resultStream = resp.GetResponseStream();
doc.Load(resultStream); // The HtmlAgilityPack
result = new Internal() { Url = url, HtmlDocument = doc };
}
catch (System.Net.WebException ex)
{
result = new WebPage.Error() { Url = url, Exception = ex };
}
catch (Exception ex)
{
ex.Data.Add("Url", url); // Annotate the exception with the Url
throw;
}
// Success, hand off the page
yield return new WebPage.Internal() { Url = url, HtmlDocument = doc };
// And and now queue up all the links on this page
foreach (HtmlNode link in doc.DocumentNode.SelectNodes(@"//a[@href]"))
{
HtmlAttribute att = link.Attributes["href"];
if (att == null) continue;
string href = att.Value;
if (href.StartsWith("javascript", StringComparison.InvariantCultureIgnoreCase)) continue; // ignore javascript on buttons using a tags
Uri urlNext = new Uri(href, UriKind.RelativeOrAbsolute);
// Make it absolute if it's relative
if (!urlNext.IsAbsoluteUri)
{
urlNext = new Uri(urlRoot, urlNext);
}
if (!allSiteUrls.Contains(urlNext))
{
allSiteUrls.Add(urlNext); // keep track of every page we've handed off
if (urlRoot.IsBaseOf(urlNext))
{
queue.Enqueue(urlNext);
}
else
{
yield return new WebPage.External() { Url = urlNext };
}
}
}
}
}
}
public static string GetStyleSheetFileForUrl(string location, Project project)
{
//TODO: This needs to expand bundles, convert urls to local file names, and move from .min.css files to .css files where applicable
//NOTE: Project parameter here is for the discovery of linked files, ones that might exist outside of the project structure
var projectPath = project.Properties.Item("FullPath").Value.ToString();
if (!projectPath.EndsWith(Path.DirectorySeparatorChar + "", StringComparison.OrdinalIgnoreCase))
{
projectPath += Path.DirectorySeparatorChar;
}
var projectUri = new Uri(projectPath, UriKind.Absolute);
if (location == null)
{
return null;
}
var locationUri = new Uri(location, UriKind.RelativeOrAbsolute);
//No absolute paths, unless they map into the same project
if (locationUri.IsAbsoluteUri)
{
if (projectUri.IsBaseOf(locationUri))
{
locationUri = locationUri.MakeRelativeUri(projectUri);
}
else
{
//TODO: Fix this, it'll only work if the site is at the root of the server as is
locationUri = new Uri(locationUri.LocalPath, UriKind.Relative);
}
if (locationUri.IsAbsoluteUri)
{
return null;
}
}
var locationUrl = locationUri.ToString().TrimStart('/').ToLowerInvariant();
//Hoist .min.css -> .css
if (locationUrl.EndsWith(".min.css", StringComparison.OrdinalIgnoreCase))
{
locationUrl = locationUrl.Substring(0, locationUrl.Length - 8) + ".css";
}
locationUri = new Uri(locationUrl, UriKind.Relative);
string filePath;
try
{
Uri realLocation;
if (Uri.TryCreate(projectUri, locationUri, out realLocation) && File.Exists(realLocation.LocalPath))
{
//Try to move from .css -> .less
var lessFile = Path.ChangeExtension(realLocation.LocalPath, ".less");
if (File.Exists(lessFile))
{
locationUri = new Uri(lessFile, UriKind.Relative);
Uri.TryCreate(projectUri, locationUri, out realLocation);
}
filePath = realLocation.LocalPath;
}
else
{
//Try to move from .min.css -> .less
var lessFile = Path.ChangeExtension(realLocation.LocalPath, ".less");
if (!File.Exists(lessFile))
{
return null;
}
locationUri = new Uri(lessFile, UriKind.Relative);
Uri.TryCreate(projectUri, locationUri, out realLocation);
filePath = realLocation.LocalPath;
}
}
catch (IOException)
{
return null;
}
return filePath;
}
/// <inheritdoc />
public override bool IsOn(Uri location)
{
var value = base.IsOn(location);
if (value)
{
return true;
}
var additionalAddress = new Uri(Config.BaseWebAddress, "form");
// We need additional logic to test other routes that MVC could use
return additionalAddress.IsBaseOf(location);
}
/// <summary>
/// Returns a hash set of operation imports (actions and functions) in the given entry.
/// </summary>
/// <param name="entry">The entry in question.</param>
/// <param name="model">The edm model to resolve operation imports.</param>
/// <param name="metadataDocumentUri">The metadata document uri.</param>
/// <returns>The hash set of operation imports (actions and functions) in the given entry.</returns>
private static HashSet<IEdmOperation> GetOperationsInEntry(ODataEntry entry, IEdmModel model, Uri metadataDocumentUri)
{
Debug.Assert(entry != null, "entry != null");
Debug.Assert(model != null, "model != null");
Debug.Assert(metadataDocumentUri != null && metadataDocumentUri.IsAbsoluteUri, "metadataDocumentUri != null && metadataDocumentUri.IsAbsoluteUri");
HashSet<IEdmOperation> edmOperationImportsInEntry = new HashSet<IEdmOperation>(EqualityComparer<IEdmOperation>.Default);
IEnumerable<ODataOperation> operations = ODataUtilsInternal.ConcatEnumerables((IEnumerable<ODataOperation>)entry.NonComputedActions, (IEnumerable<ODataOperation>)entry.NonComputedFunctions);
if (operations != null)
{
foreach (ODataOperation operation in operations)
{
Debug.Assert(operation.Metadata != null, "operation.Metadata != null");
string operationMetadataString = UriUtils.UriToString(operation.Metadata);
Debug.Assert(
ODataJsonLightUtils.IsMetadataReferenceProperty(operationMetadataString),
"ODataJsonLightUtils.IsMetadataReferenceProperty(operationMetadataString)");
Debug.Assert(
operationMetadataString[0] == ODataConstants.ContextUriFragmentIndicator || metadataDocumentUri.IsBaseOf(operation.Metadata),
"operationMetadataString[0] == JsonLightConstants.ContextUriFragmentIndicator || metadataDocumentUri.IsBaseOf(operation.Metadata)");
string fullyQualifiedOperationName = ODataJsonLightUtils.GetUriFragmentFromMetadataReferencePropertyName(metadataDocumentUri, operationMetadataString);
IEnumerable<IEdmOperation> edmOperations = model.ResolveOperations(fullyQualifiedOperationName);
if (edmOperations != null)
{
foreach (IEdmOperation edmOperation in edmOperations)
{
edmOperationImportsInEntry.Add(edmOperation);
}
}
}
}
return edmOperationImportsInEntry;
}
public static WebGraph GetGraph(Uri urlRoot)
{
var graph = new WebGraph();
var queue = new Queue<Uri>();
var allSiteUrls = new HashSet<Uri>();
queue.Enqueue(urlRoot);
allSiteUrls.Add(urlRoot);
while (queue.Count > 0)
{
Uri url = queue.Dequeue();
HttpWebRequest oReq = (HttpWebRequest)WebRequest.Create(url);
oReq.UserAgent = @"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5";
var httpResponse = (HttpWebResponse)oReq.GetResponse();
WebPage result;
if (httpResponse.ContentType.StartsWith("text/html", StringComparison.InvariantCultureIgnoreCase))
{
var htmlDocument = new HtmlDocument();
try
{
var resultStream = httpResponse.GetResponseStream();
htmlDocument.Load(resultStream); // The HtmlAgilityPack
result = new Internal() { Url = url, HtmlDocument = htmlDocument };
}
catch (WebException ex)
{
result = new WebPage.Error() { Url = url, Exception = ex };
}
catch (Exception ex)
{
ex.Data.Add("Url", url);
throw;
}
string vertexUrl = url.ToString().Replace(urlRoot.ToString(), "/").Replace("//", "/");
WebVertex v1 = graph.Vertices.SingleOrDefault(v => v.Url == vertexUrl);
if (v1 == null)
{
v1 = new WebVertex(vertexUrl);
graph.AddVertex(v1);
}
var links = htmlDocument.DocumentNode.SelectNodes(@"//a[@href]");
if (links != null)
{
foreach (HtmlNode link in links)
{
HtmlAttribute att = link.Attributes["href"];
if (att == null) continue;
string href = att.Value;
if (href.StartsWith("javascript", StringComparison.InvariantCultureIgnoreCase)) continue; // ignore javascript on buttons using a tags
Uri urlNext = new Uri(href, UriKind.RelativeOrAbsolute);
Uri urlNext2 = new Uri(href, UriKind.RelativeOrAbsolute);
// Make it absolute if it's relative
if (!urlNext.IsAbsoluteUri)
{
urlNext = new Uri(Path.Combine(urlRoot.AbsoluteUri, urlNext.ToString()));
}
if (!allSiteUrls.Contains(urlNext))
{
allSiteUrls.Add(urlNext); // keep track of every page we've handed off
if (urlRoot.IsBaseOf(urlNext))
{
queue.Enqueue(urlNext);
}
string vertexUrl2 = urlNext2.ToString();
if (!vertexUrl2.StartsWith("http://") && !vertexUrl2.StartsWith("/"))
{
vertexUrl2 = "/" + vertexUrl2;
}
WebVertex v2 = graph.Vertices.SingleOrDefault(v => v.Url == vertexUrl2);
if (v2 == null)
{
v2 = new WebVertex(vertexUrl2);
graph.AddVertex(v2);
}
graph.AddEdge(new WebEdge(v1, v2));
}
}
}
}
}
return graph;
}
List<object> arrUrlStr = new List<object>(); //List of URLs
#endregion Fields
#region Methods
public void findUri(Uri urlRoot, ref object result, ref int progress)
{
var queue = new Queue<Uri>();
result = "Root url is :" + urlRoot;
progress += 1;
Thread.Sleep(1000);
queue.Enqueue(urlRoot);
arrUrlStr.Add(urlRoot);
//get the base url
Uri uri = new Uri(urlRoot.ToString());
string baseUrl = uri.GetLeftPart(UriPartial.Authority);
string hostName = uri.Host;
string Rstring;
while (queue.Count > 0)
{
Uri url = queue.Dequeue();
HttpWebRequest HttpWReq = (HttpWebRequest)WebRequest.Create(url); // Create a request for the URL.
HttpWebResponse HttpWResp;
try
{
HttpWResp = (HttpWebResponse)HttpWReq.GetResponse(); //Get the response.
Stream ReceiveStream = HttpWResp.GetResponseStream(); //Get the stream associated with the response.
StreamReader readStream = new StreamReader(ReceiveStream); //Pipes the stream to a higher level stream reader.
Rstring = readStream.ReadToEnd(); //Reads it to the end.
//URL regular expression in C#
Regex regex = new Regex("(?<=<a\\s*?href=(?:'|\"))[^'\"]*?(?=(?:'|\"))");
string href;
foreach (Match match in regex.Matches(Rstring))
{
if (!string.IsNullOrEmpty(match.Value))
{
//Discard files and documents
Regex regexObj = new Regex(@"(css|jpg|pdf|doc|docx|ppt|pptx|js|png|ico|zip)");
Match matchResult = regexObj.Match(match.Value);
if (matchResult.Success)
{
continue;
}
try
{
Uri tempUrl = new Uri(match.Value, UriKind.RelativeOrAbsolute);
if (urlRoot.IsBaseOf(tempUrl))
{
href = "";
if (!match.Value.Contains(baseUrl))
{
href = baseUrl + "/" + match.Value;
}
else
{
href = match.Value;
}
Uri urlNext = new Uri(href, UriKind.RelativeOrAbsolute);
if (!arrUrlStr.Contains(urlNext))
{
arrUrlStr.Add(urlNext);
queue.Enqueue(urlNext);
result = urlNext.ToString();
progress += 1;
Thread.Sleep(1000);
//Console.WriteLine(urlNext.ToString());
urlNum++;
}
}
}
catch (Exception)//UriFormatException
{
Console.WriteLine("INVALID URI!!");
}
}
}
// Clean up the streams and the response.
ReceiveStream.Close();
readStream.Close();
HttpWResp.Close();
}
catch (WebException ex)
{
HttpWResp = ex.Response as HttpWebResponse;
result = ex.Message;
progress += 1;
//Console.WriteLine("HELL!!! Response Error 404 happened!!");
}
}
result = string.Format("Crawling finished!!Total number of URl is {0}", urlNum);
}
public void IsBaseOf_Null ()
{
Uri http = new Uri ("http://www.mono-project.com/Main_Page#FAQ?Edit");
try {
http.IsBaseOf (null);
Assert.Fail ();
} catch (NullReferenceException) {
}
}
/// <summary>
/// Append a relative path to a URI, handling trailing slashes appropriately.
/// </summary>
/// <param name="uri">The base URI.</param>
/// <param name="relativeUri">The relative or absolute URI.</param>
/// <param name="sep">The separator.</param>
/// <returns>The appended Uri.</returns>
internal static Uri AppendPathToSingleUri(Uri uri, string relativeUri, string sep)
{
if (uri == null || relativeUri.Length == 0)
{
return uri;
}
Uri absoluteUri;
// Because of URI's Scheme, URI.TryCreate() can't differentiate a string with colon from an absolute URI.
// A workaround is added here to verify if a given string is an absolute URI.
if (Uri.TryCreate(relativeUri, UriKind.Absolute, out absoluteUri) && (string.CompareOrdinal(absoluteUri.Scheme, "http") == 0 || string.CompareOrdinal(absoluteUri.Scheme, "https") == 0))
{
// Handle case if relPath is an absolute Uri
if (uri.IsBaseOf(absoluteUri))
{
return absoluteUri;
}
else
{
// Happens when using fiddler, DNS aliases, or potentially NATs
absoluteUri = new Uri(relativeUri);
return new Uri(uri, absoluteUri.AbsolutePath);
}
}
sep = Uri.EscapeUriString(sep);
relativeUri = Uri.EscapeUriString(relativeUri);
UriBuilder ub = new UriBuilder(uri);
string appendString = null;
if (ub.Path.EndsWith(sep, StringComparison.Ordinal))
{
appendString = relativeUri;
}
else
{
appendString = sep + relativeUri;
}
ub.Path += appendString;
return ub.Uri;
}
public void Filex_Methods ()
{
Uri uri = new Uri ("filex:///readme.txt");
Assert.AreEqual ("readme.txt", uri.GetComponents (UriComponents.Path, UriFormat.SafeUnescaped), "GetComponents");
Assert.IsTrue (uri.IsBaseOf (uri), "IsBaseOf");
Assert.IsTrue (uri.IsWellFormedOriginalString (), "IsWellFormedOriginalString");
// ??? our parser doesn't seems to be called :(
}
public void Generic_Methods ()
{
Uri uri = new Uri ("generic://www.mono-project.com/");
Assert.AreEqual (String.Empty, uri.GetComponents (UriComponents.Path, UriFormat.SafeUnescaped), "GetComponents");
Assert.IsTrue (uri.IsBaseOf (uri), "IsBaseOf");
Assert.IsTrue (uri.IsWellFormedOriginalString (), "IsWellFormedOriginalString");
}
public void TestDeep(Cookie cookie, SqlConnectionInfo sci)
{
Uri root = new Uri("http://localhost/DBlog/Default.aspx");
List<Uri> queue = new List<Uri>(2500);
List<Uri> visited = new List<Uri>(5000);
PerformanceDataCollection perfdata = (sci == null ? null : new PerformanceDataCollection());
double totaltime = 0;
int totalcount = 0;
queue.Add(root);
Dictionary<Uri, Uri> references = new Dictionary<Uri, Uri>();
references.Add(root, new Uri("http://localhost/DBlog/"));
while (queue.Count > 0)
{
Uri topofqueue = queue[0];
List<HtmlUri> links;
double ts = 0;
PerformanceData perf = (perfdata != null ? new PerformanceData(topofqueue.ToString()) : null);
try
{
TraceServer traceServerReader = (sci == null ? null : SqlTrace.BeginTrace(sci));
TestPage(references[topofqueue], topofqueue, cookie, out links, out ts);
if (perfdata != null) perfdata.Add(SqlTrace.EndTrace(topofqueue.ToString(), traceServerReader));
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine("{0}: {1}", topofqueue, ex.Message);
throw ex;
}
finally
{
totalcount++;
totaltime += ts;
// add to visited links
visited.Add(topofqueue);
// remove from queue
queue.RemoveAt(0);
}
if (perfdata != null)
{
perfdata.Add(perf);
}
// -----------------
Console.Write("{0}/{1} [avg: {2}]", totalcount, queue.Count, (totaltime / totalcount).ToString("0.00"));
if (perf != null) Console.Write("[SQL: {0} in {1}] ", perf.Queries, perf.TotalDuration);
Console.Write(" => ");
int count = 0;
foreach (HtmlUri uri in links)
{
Uri fulluri = uri.Uri;
if (!root.IsBaseOf(fulluri))
{
// Console.WriteLine("\tSkipping {0}.", uri);
continue;
}
if (references.ContainsKey(fulluri) || queue.Contains(fulluri) || visited.Contains(fulluri))
continue;
Assert.IsFalse(fulluri.ToString().Contains("\0"),
string.Format("Uri {0} in {1} contains non-ASCII character.", fulluri, topofqueue));
Assert.IsFalse(fulluri.ToString().Contains("<%"),
string.Format("Uri {0} in {1} contains non-executed ASP.NET code.", fulluri, topofqueue));
Assert.IsFalse(fulluri.ToString().Contains("id=0"),
string.Format("Uri {0} in {1} contains a link to a zero id.", fulluri, topofqueue));
references.Add(fulluri, topofqueue);
// Console.WriteLine("\tAdding {0}.", fulluri.OriginalString);
queue.Add(fulluri);
count++;
}
if ((perfdata != null) && (((totalcount > 0) && ((totalcount % 100) == 0)) || (queue.Count == 0)))
{
perfdata.DumpPigs();
}
}
}
public void IsBaseOf_Null ()
{
Uri http = new Uri ("http://www.mono-project.com/Main_Page#FAQ?Edit");
try {
http.IsBaseOf (null);
Assert.Fail ();
}
#if NET_4_0
catch (ArgumentNullException) {
}
#else
catch (NullReferenceException) {
}
#endif
}
/// <summary>
/// This routine was implemented to assist with finding credentials in the settings file using the uri
/// It appears the package uri is used to find the credentials ex) https://hostname/api/nuget/Download/packagename/versionnumber
/// but the settings file will more than likely only have the repository uri ex) https://hostname/api/nuget
/// This routine will attempt to find the uri in settings as is
/// If not found, see if source Uri is base of uri
/// </summary>
/// <param name="uri1">base or source URI</param>
/// <param name="uri2">full URI</param>
/// <returns></returns>
public static bool UriStartsWith(Uri uri1, Uri uri2)
{
return UriUtility.UriEquals(uri1, uri2) || uri1.IsBaseOf(uri2);
}
private IEnumerable<string> GetFiles(IEnumerable<string> locations)
{
var project = Connection.Project;
//TODO: This needs to expand bundles, convert urls to local file names, and move from .min.css files to .css files where applicable
//NOTE: Project parameter here is for the discovery of linked files, ones that might exist outside of the project structure
var projectPath = project.Properties.Item("FullPath").Value.ToString();
var projectUri = new Uri(projectPath, UriKind.Absolute);
foreach (var location in locations)
{
if (location == null)
{
continue;
}
var locationUri = new Uri(location, UriKind.RelativeOrAbsolute);
//No absolute paths, unless they map into the same project
if (locationUri.IsAbsoluteUri)
{
if (projectUri.IsBaseOf(locationUri))
{
locationUri = locationUri.MakeRelativeUri(projectUri);
}
else
{
//TODO: Fix this, it'll only work if the site is at the root of the server as is
locationUri = new Uri(locationUri.LocalPath, UriKind.Relative);
}
if (locationUri.IsAbsoluteUri)
{
continue;
}
}
var locationUrl = locationUri.ToString().TrimStart('/').ToLowerInvariant();
//Hoist .min.css -> .css
if (locationUrl.EndsWith(".min.css"))
{
locationUrl = locationUrl.Substring(0, locationUrl.Length - 8) + ".css";
}
locationUri = new Uri(locationUrl, UriKind.Relative);
string filePath;
try
{
Uri realLocation;
if (Uri.TryCreate(projectUri, locationUri, out realLocation) && File.Exists(realLocation.LocalPath))
{
//Try to move from .css -> .less
var lessFile = Path.ChangeExtension(realLocation.LocalPath, ".less");
if (File.Exists(lessFile))
{
locationUri = new Uri(lessFile, UriKind.Relative);
Uri.TryCreate(projectUri, locationUri, out realLocation);
}
filePath = realLocation.LocalPath;
}
else
{
//Try to move from .min.css -> .less
var lessFile = Path.ChangeExtension(realLocation.LocalPath, ".less");
if (!File.Exists(lessFile))
{
continue;
}
locationUri = new Uri(lessFile, UriKind.Relative);
Uri.TryCreate(projectUri, locationUri, out realLocation);
filePath = realLocation.LocalPath;
}
}
catch (IOException)
{
continue;
}
yield return filePath;
}
}
/// <summary>
/// Check whether the <paramref name="baseUri"/> Uri is the base of the <paramref name="uri"/> Uri.
/// </summary>
/// <param name="baseUri">The candidate base Uri.</param>
/// <param name="uri">The Uri to check.</param>
/// <returns>True if the <paramref name="baseUri"/> is the base of the <paramref name="uri"/> Uri.</returns>
private static bool IsBaseOf(Uri baseUri, Uri uri)
{
return baseUri.IsBaseOf(uri);
}
