// Add value to list
private static bool AppendValue(string value, List <string> domains)
{
string domain = value
.Replace("www.", "").ToLower();
// Remove search results
if (
domain.Contains("google") ||
domain.Contains("bing") ||
domain.Contains("yandex") ||
domain.Contains("duckduckgo"))
{
return(false);
}
// If cookie value
if (domain
.StartsWith("."))
{
domain = domain.Substring(1);
}
// Get hostname from url
try {
domain = new System.Uri(domain).Host;
} catch (System.UriFormatException) { }
// Remove .com, .org
domain = System.IO.Path.GetFileNameWithoutExtension(domain);
domain = domain.Replace(".com", "").Replace(".org", "");
// Check if domain already exists in list
foreach (string domainValue in domains)
{
if (domain.ToLower().Replace(" ", "").Contains(domainValue.ToLower().Replace(" ", "")))
{
return(false);
}
}
// Convert first char to upper-case
domain = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ToTitleCase(domain);
domains.Add(domain);
return(true);
}
public void PutExistingBlobControllerTest()
{
string blobUri = _ctx.GetBlobUri("test.txt");
var pipelineResponse = BlobPipelineTests.BlobRequest("GET", blobUri);
string redirectHost = new Uri(pipelineResponse.Location).Host;
var content = new StringContent("hello world", System.Text.Encoding.UTF8, "text/plain");
content.Headers.Add("x-ms-version", "2013-08-15");
content.Headers.Add("x-ms-date", "Wed, 23 Oct 2013 22:33:355 GMT");
content.Headers.Add("x-ms-blob-content-disposition", "attachment; filename=\"test.txt\"");
content.Headers.Add("x-ms-blob-type", "BlockBlob");
content.Headers.Add("x-ms-meta-m1", "v1");
content.Headers.Add("x-ms-meta-m2", "v2");
var response = _ctx.Runner.ExecuteRequest(blobUri,
"PUT",
content,
HttpStatusCode.Created);
// Validate that the blob is still in the same account
pipelineResponse = BlobPipelineTests.BlobRequest("GET", blobUri);
Assert.AreEqual(redirectHost.ToLower(), new Uri(pipelineResponse.Location).Host.ToLower());
}
/// <summary>
/// Performs a SaveAs operation of an open document. Called from SaveItem after the running document table has been updated with the new doc data.
/// </summary>
/// <param name="docData">A pointer to the document in the rdt</param>
/// <param name="newFilePath">The new file path to the document</param>
/// <returns></returns>
protected override int AfterSaveItemAs(IntPtr docData, string newFilePath)
{
if (String.IsNullOrEmpty(newFilePath))
{
throw new ArgumentException(SR.GetString(SR.ParameterCannotBeNullOrEmpty), "newFilePath");
}
int returnCode = VSConstants.S_OK;
newFilePath.Trim();
//Identify if Path or FileName are the same for old and new file
string newDirectoryName = Path.GetDirectoryName(newFilePath);
Uri newDirectoryUri = new Uri(newDirectoryName);
string newCanonicalDirectoryName = newDirectoryUri.LocalPath;
newCanonicalDirectoryName = newCanonicalDirectoryName.TrimEnd(Path.DirectorySeparatorChar);
string oldCanonicalDirectoryName = new Uri(Path.GetDirectoryName(this.GetMkDocument())).LocalPath;
oldCanonicalDirectoryName = oldCanonicalDirectoryName.TrimEnd(Path.DirectorySeparatorChar);
string errorMessage = String.Empty;
bool isSamePath = NativeMethods.IsSamePath(newCanonicalDirectoryName, oldCanonicalDirectoryName);
bool isSameFile = NativeMethods.IsSamePath(newFilePath, this.Url);
// Currently we do not support if the new directory is located outside the project cone
string projectCannonicalDirecoryName = new Uri(this.ProjectMgr.ProjectFolder).LocalPath;
projectCannonicalDirecoryName = projectCannonicalDirecoryName.TrimEnd(Path.DirectorySeparatorChar);
if (!isSamePath && newCanonicalDirectoryName.ToLower().IndexOf(projectCannonicalDirecoryName.ToLower()) == -1)
{
errorMessage = String.Format(CultureInfo.CurrentCulture, SR.GetString(SR.LinkedItemsAreNotSupported), Path.GetFileNameWithoutExtension(newFilePath));
throw new InvalidOperationException(errorMessage);
}
//Get target container
HierarchyNode targetContainer = null;
if (isSamePath)
{
targetContainer = this.Parent;
}
else if (NativeMethods.IsSamePath(newCanonicalDirectoryName, projectCannonicalDirecoryName))
{
//the projectnode is the target container
targetContainer = this.ProjectMgr;
}
else
{
//search for the target container among existing child nodes
targetContainer = this.ProjectMgr.FindChild(newDirectoryName);
if (targetContainer != null && (targetContainer is FileNode))
{
// We already have a file node with this name in the hierarchy.
errorMessage = String.Format(CultureInfo.CurrentCulture, SR.GetString(SR.FileAlreadyExistsAndCannotBeRenamed), Path.GetFileNameWithoutExtension(newFilePath));
throw new InvalidOperationException(errorMessage);
}
}
if (targetContainer == null)
{
// Add a chain of subdirectories to the project.
string relativeUri = PackageUtilities.GetPathDistance(this.ProjectMgr.BaseURI.Uri, newDirectoryUri);
Debug.Assert(!String.IsNullOrEmpty(relativeUri) && relativeUri != newDirectoryUri.LocalPath, "Could not make pat distance of " + this.ProjectMgr.BaseURI.Uri.LocalPath + " and " + newDirectoryUri);
targetContainer = this.ProjectMgr.CreateFolderNodes(relativeUri);
}
Debug.Assert(targetContainer != null, "We should have found a target node by now");
//Suspend file changes while we rename the document
string oldrelPath = this.ItemNode.GetMetadata(ProjectFileConstants.Include);
string oldName = Path.Combine(this.ProjectMgr.ProjectFolder, oldrelPath);
SuspendFileChanges sfc = new SuspendFileChanges(this.ProjectMgr.Site, oldName);
sfc.Suspend();
try
{
// Rename the node.
DocumentManager.UpdateCaption(this.ProjectMgr.Site, Path.GetFileName(newFilePath), docData);
// Check if the file name was actually changed.
// In same cases (e.g. if the item is a file and the user has changed its encoding) this function
// is called even if there is no real rename.
if (!isSameFile || (this.Parent.ID != targetContainer.ID))
{
// The path of the file is changed or its parent is changed; in both cases we have
// to rename the item.
this.RenameFileNode(oldName, newFilePath, targetContainer.ID);
OnInvalidateItems(this.Parent);
}
}
catch (Exception e)
{
Trace.WriteLine("Exception : " + e.Message);
this.RecoverFromRenameFailure(newFilePath, oldrelPath);
throw e;
}
finally
{
sfc.Resume();
}
return returnCode;
}
public void TestNoShadow()
{
Assembly assembly = GetType().Assembly;
string localPath = new Uri(assembly.CodeBase).LocalPath;
string location = assembly.Location;
Assert.IsTrue(localPath.ToLower() != location.ToLower(), "assembly at '" + location + "' is being shadow copied");
}
public static bool IsBaseOfWithPath(this Uri baseUri, Uri uri)
{
// Note: Uri.IsBaseOf returns true also when called with
// new Uri("http://contoso.com/path").IsBaseOf(new Uri("http://contoso.com"))
// Which for me is wrong...so I check also the path
if (string.IsNullOrWhiteSpace(baseUri.ToString()))
return false;
if (baseUri.IsAbsoluteUri && uri.IsAbsoluteUri)
{
baseUri = baseUri.ToLower();
uri = uri.ToLower();
if (baseUri.Scheme == uri.Scheme &&
baseUri.Host == uri.Host &&
baseUri.Port == uri.Port &&
uri.GetComponents(UriComponents.Path, UriFormat.UriEscaped).StartsWith(baseUri.GetComponents(UriComponents.Path, UriFormat.UriEscaped).Trim('/')))
return true;
else
return false;
}
else if (!baseUri.IsAbsoluteUri && !uri.IsAbsoluteUri)
{
baseUri = baseUri.RemoveTrailingSlash();
return uri.ToString().StartsWith(baseUri.ToString(), StringComparison.InvariantCultureIgnoreCase);
}
else
throw new XrcException("Uri not valid, cannot compare a relative uri with an absolute uri.");
}
/// <summary>
/// Returns a value indicating whether the original file is new or has been updated.
/// </summary>
/// <returns>
/// True if the the original file is new or has been updated; otherwise, false.
/// </returns>
internal async Task<bool> IsNewOrUpdatedFileAsync()
{
string path = this.CachedPath;
bool isUpdated = false;
CachedImage cachedImage;
// is this a cropup url ....
bool isCropup = this.requestPath.ToLower().IndexOf("/cropup/") > -1;
if (isCropup)
{
try
{
// get the json file name associated with the request. We will do timestamp check against that.
string urlpath = new Uri(this.requestPath).PathAndQuery;
string servermappedpath = request.MapPath(urlpath.Substring(urlpath.ToLower().IndexOf("/media")));
string jsonFileToCheck = servermappedpath + ".crop.json";
cachedImage = await CacheIndexer.GetValueAsync(path);
FileInfo jsonFileInfo = new FileInfo(jsonFileToCheck);
if (cachedImage != null && jsonFileInfo.Exists)
{
// Looking for json file
// or if the max age is different.
if (this.IsExpired(cachedImage.CreationTimeUtc) || jsonFileInfo.LastWriteTimeUtc > cachedImage.LastWriteTimeUtc)
{
CacheIndexer.Remove(path);
isUpdated = true;
}
}
else if (!jsonFileInfo.Exists)
{
// no json means no first time save means no changes
isUpdated = true;
}
else
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
}
catch
{
isUpdated = false;
}
}
else if (this.isRemote)
{
cachedImage = await CacheIndexer.GetValueAsync(path);
if (cachedImage != null)
{
// Can't check the last write time so check to see if the cached image is set to expire
// or if the max age is different.
if (this.IsExpired(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(path);
isUpdated = true;
}
}
else
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
}
else
{
// Test now for locally requested files.
cachedImage = await CacheIndexer.GetValueAsync(path);
if (cachedImage != null)
{
FileInfo imageFileInfo = new FileInfo(this.requestPath);
if (imageFileInfo.Exists)
{
// Pull the latest info.
imageFileInfo.Refresh();
// Check to see if the last write time is different of whether the
// cached image is set to expire or if the max age is different.
if (!this.RoughDateTimeCompare(imageFileInfo.LastWriteTimeUtc, cachedImage.LastWriteTimeUtc)
|| this.IsExpired(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(path);
isUpdated = true;
}
}
}
else
{
// Nothing in the cache so we should return true.
isUpdated = true;
}
}
return isUpdated;
}
public LinkGrap(string link)
{
LinkGrap Item = null;
string saveLocation = HostingEnvironment.MapPath("~/lib/u/");
Item = (LinkGrap)HttpContext.Current.Cache[string.Format(cacheKey, link)];
if (Item == null)
{
#region xử lý Link
HttpWebRequest wrq;
wrq = (HttpWebRequest)(WebRequest.Create(link));
string host = new Uri(link).Host;
wrq.Credentials = CredentialCache.DefaultCredentials;
wrq.Method = "GET";
wrq.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; vi; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3";
wrq.SendChunked = true;
if (link.IndexOf("zing.vn") != -1)
{
wrq.Referer = "http://mp3.zing.vn";
}
try
{
HttpWebResponse wrp = (HttpWebResponse)wrq.GetResponse();
HtmlDocument doc = new HtmlDocument();
contentType = wrp.ContentType;
if (contentType.ToLower().IndexOf("html") != -1)
{
doc.Load(wrp.GetResponseStream(), Encoding.UTF8);
string domain = "http://" + (new Uri(link)).Host;
#region Title
if (doc.DocumentNode.SelectNodes("//title | //TITLE") != null)
{
HtmlNode titleNode = doc.DocumentNode.SelectNodes("//title | //TITLE")[0];
Title = titleNode.InnerText;
}
#endregion
#region Desc
if (doc.DocumentNode.SelectNodes("//meta[@name='description'] | //meta[@name='DESCRIPTION']") != null)
{
HtmlNode titleNode = doc.DocumentNode.SelectNodes("//meta[@name='description'] | //meta[@name='DESCRIPTION']")[0];
Description = titleNode.Attributes["content"].Value;
}
#endregion
#region Content
ContentRawHtml = doc.DocumentNode.InnerHtml;
ContentRawText = doc.DocumentNode.InnerText;
Content = Wrapper(host, link, doc);
#endregion
#region Images
//if (doc.DocumentNode.SelectNodes("//img | //IMG") != null)
//{
// List<string> _list = new List<string>();
// foreach (HtmlNode _img in doc.DocumentNode.SelectNodes("//img | //IMG"))
// {
// if (_img.Attributes["src"] != null)
// {
// string src = _img.Attributes["src"].Value;
// if (src.ToLower().IndexOf("http://") != 0)
// {
// src = domain + src;
// }
// _list.Add(src);
// }
// }
// Images = _list;
//}
#endregion
#region Link
List<string> _inLink = new List<string>();
List<string> _outLink = new List<string>();
foreach (HtmlNode a in doc.DocumentNode.SelectNodes("//a|//A"))
{
string href = string.Empty;
if (a.Attributes["href"] != null)
{
href = a.Attributes["href"].Value;
}
else if (a.Attributes["HREF"] != null)
{
href = a.Attributes["HREF"].Value;
}
if (!string.IsNullOrEmpty(href))
{
if (href.ToLower().IndexOf("javascrip") != 0 && href.ToLower().IndexOf("#") != 0)
{
if (href.ToLower().IndexOf("http://") == 0)//HTTPLink
{
Uri _href = new Uri(href);
if (_href.Host.ToLower().IndexOf(host.ToLower()) != -1)
{
_inLink.Add(href);
}
else
{
_outLink.Add(href);
}
}
else
{
if (href.ToLower().IndexOf("/") == -1) href = "/" + href;
if (href.ToLower().IndexOf("../") == 0) href = href.Substring(href.LastIndexOf("../") + 3);
href = "http://" + host + href;
_inLink.Add(href);
}
}
}
InLink = _inLink;
OutLink = _outLink;
}
#endregion
}
#region Ảnh
//else
//{
// if (contentType.IndexOf("image") != -1)
// {
// Title = link;
// Description = link;
// List<string> _list = new List<string>();
// ImageProcess gimg = new ImageProcess(new Uri(link), link);
// if (gimg.Width > 100 && gimg.Heigth > 75)
// {
// _list.Add(link);
// }
// Images = _list;
// }
//}
#endregion
HttpContext.Current.Cache.Insert(string.Format(cacheKey, link), this);
}
catch (WebException ex)
{
LinkGrap _obj2 = new LinkGrap();
_obj2.Title = "0";
HttpContext.Current.Cache.Remove(string.Format(cacheKey, link));
}
#endregion
}
}
///<summary>
///</summary>
///<returns>true of successful; false if failed</returns>
public virtual bool SetRepositoryAddress()
{
var repo = new HgRepository(TargetDestination, _progress);
var name = new Uri(URL).Host;
if (String.IsNullOrEmpty(name)) //This happens for repos on the local machine
{
name = @"LocalRepository";
}
if (name.ToLower().Contains(@"languagedepot"))
name = @"LanguageDepot";
var address = RepositoryAddress.Create(name, URL);
//this will also remove the "default" one that hg puts in, which we don't really want.
repo.SetKnownRepositoryAddresses(new[] { address });
repo.SetIsOneDefaultSyncAddresses(address, true);
return true;
}
/// <summary>
/// Tests ALL link suggestions in LS_DEBUG mode. Should only be used for small caches.
/// </summary>
/// <param name="proxy">The local proxy.</param>
public static void DebugLinkSuggestion(RCLocalProxy proxy)
{
CacheManager cm = proxy.ProxyCacheManager;
string[] stopwords = File.ReadAllLines("stopwords.txt");
List<string> sites = cm.TextFiles();
int i = 1;
foreach (string site in sites)
{
proxy.Logger.Debug(String.Format("Working site {0} of {1}", i++, sites.Count));
string url = cm.AbsoluteFilePathToUri(site);
string content = Utils.ReadFileAsString(site);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(content);
HtmlNode head = doc.DocumentNode.SelectSingleNode("/html/head");
HtmlNode body = doc.DocumentNode.SelectSingleNode("/html/body");
if (head == null || body == null)
{
// We haven't sane HTML, just continue
continue;
}
// Gets all links containing text.
HtmlNodeCollection links = doc.DocumentNode.SelectNodes("//a[not(node()[2])][text()]/@href");
if (links != null)
{
int ign;
foreach (HtmlNode link in links)
{
string relTarget = link.GetAttributeValue("href", "");
string target;
try
{
target = new Uri(new Uri(url), relTarget).ToString();
}
catch (UriFormatException)
{
continue;
}
if (LINK_ANCHOR_BLACKLIST.Contains(link.InnerText.ToLower())
|| Int32.TryParse(link.InnerText, out ign) || url.ToLower().Equals(target.ToLower()))
{
// No "Here", ... links or number links
// No links to the same page! (LS_DEBUG)
continue;
}
string relFileName = CacheManager.GetRelativeCacheFileName(target, "GET");
if (!target.Equals("") && cm.IsCached(relFileName))
{
// Get anchor and surrounding text
string anchorText = link.InnerText;
string surroundingText = HtmlUtils.GetSurroundingText(link,
stopwords, false);
// Trigger LS
GetLinkSuggestions(target, url, anchorText, surroundingText,
3, proxy);
}
}
}
}
}
/// <summary>
/// Injects tooltips for every link in the HTML with meaningful text.
/// </summary>
/// <param name="html">The original page.</param>
/// <returns>The modified page.</returns>
public static string IncludeTooltips(string html, string url, CacheManager cm)
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlNode head = doc.DocumentNode.SelectSingleNode("/html/head");
HtmlNode body = doc.DocumentNode.SelectSingleNode("/html/body");
if (head == null || body == null)
{
// We haven't sane HTML, just return it as it is.
return html;
}
// include style document
HtmlNode opentipStyle = doc.CreateElement("link");
opentipStyle.SetAttributeValue("type", "text/css");
opentipStyle.SetAttributeValue("rel", "stylesheet");
opentipStyle.SetAttributeValue("href", "http://www.ruralcafe.net/css/opentip.css");
head.AppendChild(opentipStyle);
// include opentip js document
HtmlNode opentipJs = doc.CreateElement("script");
opentipJs.SetAttributeValue("type", "text/javascript");
opentipJs.SetAttributeValue("src", "http://www.ruralcafe.net/js/opentip-native.min.js");
head.AppendChild(opentipJs);
// include our js document
HtmlNode ourJs = doc.CreateElement("script");
ourJs.SetAttributeValue("type", "text/javascript");
ourJs.SetAttributeValue("src", "http://www.ruralcafe.net/js/linkSuggestion.js");
head.AppendChild(ourJs);
// include our ajax js document (Opentip ajax does not like us...)
HtmlNode ourAjaxJs = doc.CreateElement("script");
ourAjaxJs.SetAttributeValue("type", "text/javascript");
ourAjaxJs.SetAttributeValue("src", "http://www.ruralcafe.net/js/ajax.js");
head.AppendChild(ourAjaxJs);
// Include the invisible trigger element
HtmlNode trigger = doc.CreateElement("div");
trigger.SetAttributeValue("id", "rclink-trigger");
body.AppendChild(trigger);
// Gets all links containing text.
HtmlNodeCollection links = doc.DocumentNode.SelectNodes("//a[not(node()[2])][text()]/@href");
if (links != null)
{
// Modify all links
int i = 0;
int ign;
foreach (HtmlNode link in links)
{
string relTarget = link.GetAttributeValue("href", "");
string target;
try
{
target = new Uri(new Uri(url), relTarget).ToString();
}
catch (UriFormatException)
{
continue;
}
if (LINK_ANCHOR_BLACKLIST.Contains(link.InnerText.ToLower())
|| Int32.TryParse(link.InnerText, out ign) || url.ToLower().Equals(target.ToLower()))
{
// No "Here", ... links or number links
// No links to the same page! (LS_DEBUG)
continue;
}
link.SetAttributeValue("id", "rclink-" + i);
link.SetAttributeValue("onmouseover", "showSuggestions(" + i + ")");
link.SetAttributeValue("onmouseout", "clearActiveLinkNumber()");
i++;
if (Properties.Network.Default.LS_DEBUG)
{
// Highlight links pointing to downloaded docs in LS_DEBUG mode
string relFileName = CacheManager.GetRelativeCacheFileName(target, "GET");
if (!target.Equals("") && cm.IsCached(relFileName))
{
link.SetAttributeValue("class", "highlightedCachedLink");
}
}
}
}
return doc.DocumentNode.OuterHtml;
}
private void btnGo_Click(object sender, EventArgs args)
{
btnGo.Enabled = false;
miSettings.Enabled = false;
txtUrl.Enabled = false;
txtLinks.Clear();
new Thread(delegate()
{
try
{
ArrayList<string> includeList = new ArrayList<string>();
int i = 0;
foreach (ToolStripMenuItem item in miInclude.DropDownItems)
{
if (item.Checked) { includeList.Add(mRegexList[i]); }
i++;
}
ArrayList<string> excludeList = new ArrayList<string>();
i = 0;
foreach (ToolStripMenuItem item in miExclude.DropDownItems)
{
if (item.Checked) { excludeList.Add(mExcludeList[i]); }
i++;
}
Set<string> links = new Set<string>();
Uri baseUrl = new Uri(txtUrl.Text);
string html = WebUtils.GetWebPageDetectEncoding(txtUrl.Text);
foreach (string regex in includeList)
{
Regex r = new Regex(regex, RegexOptions.IgnoreCase);
Match m = r.Match(html);
while (m.Success)
{
string message = "RSS feed NOT detected.";
string url = m.Result("${rssUrl}").Trim();
url = new Uri(baseUrl, url).ToString();
string urlLower = url.ToLower();
// test whether to include link
bool ok = true;
foreach (string substr in excludeList)
{
if (urlLower.Contains(substr)) { ok = false; break; }
}
if (ok && !links.Contains(urlLower))
{
// test RSS file
if (miTestLinks.Checked)
{
string xml = null;
try { xml = WebUtils.GetWebPageDetectEncoding(url); } catch { }
bool rssXmlFound = xml != null && TestRssXml(xml);
if (rssXmlFound) { message = "RSS feed detected."; }
// convert Atom to RSS
if (xml != null && miConvertAtomToRss.Checked && !rssXmlFound && TestAtomXml(xml))
{
url = "http://www.devtacular.com/utilities/atomtorss/?url=" + HttpUtility.HtmlEncode(url);
xml = null;
try { xml = WebUtils.GetWebPageDetectEncoding(url); }
catch { }
rssXmlFound = xml != null && TestRssXml(xml);
if (rssXmlFound) { message = "RSS feed detected after converting from Atom."; }
}
else // try the format=xml trick
{
if (miFeedburnerFormat.Checked && !rssXmlFound)
{
string newUrl = url + (url.Contains("?") ? "&" : "?") + "format=xml";
try { xml = WebUtils.GetWebPageDetectEncoding(newUrl); } catch { }
rssXmlFound = xml != null && TestRssXml(xml);
if (rssXmlFound)
{
message = "RSS feed detected after applying the format=xml trick.";
url = newUrl;
}
}
}
if (miRemoveNonRss.Checked && !rssXmlFound) { Invoke(delegate() { txtLinks.Text += "#"; }); }
Invoke(delegate() { txtLinks.Text += url + "\r\n"; });
if (miOutputTestResult.Checked)
{
Invoke(delegate() { txtLinks.Text += "# " + message + "\r\n"; });
}
}
else
{
Invoke(delegate() { txtLinks.Text += url + "\r\n"; });
}
links.Add(urlLower);
}
m = m.NextMatch();
} // m.Success
}
}
catch (Exception e)
{
TryInvoke(delegate() { txtLinks.Text += e.Message + "\r\n" + e.StackTrace; });
}
finally
{
TryInvoke(delegate() { btnGo.Enabled = txtUrl.Enabled = miSettings.Enabled = true; });
}
}).Start();
}
