private static string GuessLanguage(string ImportUrl)
 {
     // guess language
     string filename = new Uri(ImportUrl).Segments.Last().ToLower();
     if (filename.StartsWith("cs_") || filename.StartsWith("csharp"))
     {
         return "CSharp";
     }
     if (filename.StartsWith("vb_") || filename.StartsWith("<Bold>B</Bold>asic"))
     {
         return "Basic";
     }
     return "";
 }
        private static string PathFromUri(string uri)
        {
            string baseUri = new Uri(uri).PathAndQuery;
            if (baseUri.StartsWith("/"))
            {
                baseUri = baseUri.Substring(1);
            }

            return baseUri;
        }
Beispiel #3
0
        /// <summary>
        /// Create a PendingCrawlItem from an href which may be relative or absolute (so need urlRoot also)
        /// </summary>
        /// <param name="urlRoot"></param>
        /// <param name="href"></param>
        /// <returns>An instance of either a PendingLocalWebPage or a PendingExternalWebPage</returns>
        public static PendingCrawlItem Factory(Uri urlRoot, Uri href, Uri referrer)
        {
            // Make it absolute if it's relative
            if (!href.IsAbsoluteUri)
            {
                href = new Uri(urlRoot, href);
            }

            if (href.Scheme == "mailto") throw new MailToException("Can't crawl a mailto " + href.ToString());

            PendingCrawlItem result;

            // used to use .isBaseOf on Uri class but that doesn't work unless the case is the same! (Windows vs Unix issue!!!)

            if (!string.Equals(href.Authority.Replace("www.",""), urlRoot.Authority.Replace("www.",""), StringComparison.InvariantCultureIgnoreCase))
            {
                // Completely different authority = off domain (modulo having a www or not on there)
                return new PendingExternalWebPage() { Uri = href, UrlRoot = urlRoot, UriReferrer = referrer };
            }

            // Is it on site and under the base directory specified?
            // NOTE: Use PATH here and NOT AbsoluteUri because we may be on www. versus non-www. domains
            // Could instead do the same www. trick as above??
            string rootMappedToNothing = new Uri(urlRoot, "").AbsolutePath;                                   // Probably empty or /
            string hrefMappedToX = new Uri(href, "X").AbsolutePath.TrimEnd("0123456789".ToArray());           // take any numbers off the end (paging)
            if (hrefMappedToX.StartsWith(rootMappedToNothing))
            {
                result = new PendingLocalWebPage() { Uri = href, UrlRoot = urlRoot, UriReferrer = referrer };
                return result;
            }

            //string cleanRoot = urlRoot.AbsolutePath;

            ////cleanRoot = ignorableTail.Replace(cleanRoot, "");

            //if (href.AbsolutePath.StartsWith(cleanRoot.TrimEnd('/'), StringComparison.InvariantCultureIgnoreCase))
            //{
            //    result = new PendingLocalWebPage() { Uri = href, UrlRoot = urlRoot, UriReferrer = referrer };
            //}
            //else
            {
                // This isn't really an ExternalWebPage, it's an Excluded Web page .. TODO: make new class here
                result = new PendingExternalWebPage() { Uri = href, UrlRoot = urlRoot, UriReferrer = referrer };
            }
            return result;
        }
 public override void LoadPlaylist(string fpath)
 {
     ItemsPaths = new ArrayList();
     //
     try
     {
         XmlReaderSettings settings = new XmlReaderSettings();
         settings.XmlResolver = null;
         settings.DtdProcessing = DtdProcessing.Ignore;
         settings.ValidationType = ValidationType.None;
         using (XmlReader reader = XmlReader.Create(fpath, settings))
         {
             CheckFileType(reader);
             while (reader.ReadToFollowing("key"))
             {
                 reader.ReadStartElement("key");
                 if (reader.ReadString().ToLower().Equals("location"))
                 {
                     reader.ReadEndElement();
                     reader.ReadStartElement("string");
                     string str = new Uri(reader.ReadString().Trim()).LocalPath;
                     if (str.StartsWith(@"\\localhost\")) str = str.Substring(12);
                     ItemsPaths.Add(str);
                     reader.ReadEndElement();
                 }
                 else
                     reader.ReadEndElement();
             }
         }
     }
     catch (IOException)
     {
         throw new Exception("Error occured during the file reading process!");
     }
     catch (XmlException)
     {
         throw new Exception("Error occured during the file parsing process!");
     }
 }
Beispiel #5
0
 public static string GetFormattedTitle(SoundPlayerInfo playerInfo)
 {
     string title = playerInfo.Name;
     if ((playerInfo.IsWeb) && (playerInfo.UserEditedName == false))
     {
         string domainName = new Uri(playerInfo.UrlOrCommandLine).Host;
         if (domainName.StartsWith("www."))
             domainName = domainName.Substring(4);
         title = domainName + ": " + playerInfo.Name;
     }
     return title;
 }
        private static string PathFromUri(string uri)
        {
            if (String.IsNullOrEmpty(uri))
            {
                return String.Empty;
            }

            string baseUri = new Uri(uri).PathAndQuery;
            if (baseUri.StartsWith("/"))
            {
                baseUri = baseUri.Substring(1);
            }

            return baseUri;
        }
        /// <summary>
        /// Create a new SourceLineNumberCollection from a URI.
        /// </summary>
        /// <param name="uri">The URI.</param>
        /// <returns>The new SourceLineNumberCollection.</returns>
        public static SourceLineNumberCollection FromUri(string uri)
        {
            if (null == uri || 0 == uri.Length)
            {
                return null;
            }

            string localPath = new Uri(uri).LocalPath;

            // make the local path really look like a normal local path
            if (localPath.StartsWith(Path.AltDirectorySeparatorChar.ToString(), StringComparison.Ordinal))
            {
                localPath = localPath.Substring(1);
            }
            localPath = localPath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);

            return new SourceLineNumberCollection(localPath);
        }
Beispiel #8
0
        private async Task<List<ImageLink>> GetImgurImageOrAlbumFromUrl(Post post, string outputPath, CancellationToken token)
        {
            token.ThrowIfCancellationRequested();
            var links = new List<ImageLink>();

            OutputLine($"\tGetting links from post: {post.Title}", true);

            var url = new Uri(post.Url.ToString()).GetLeftPart(UriPartial.Path).Replace(".gifv", ".gif");

            url = url.StartsWith("http://") ? url : "http://" + url.Substring(url.IndexOf(post.Url.DnsSafeHost, StringComparison.Ordinal));

            var name = Path.GetInvalidFileNameChars().Aggregate(post.AuthorName, (current, c) => current.Replace(c, '-'));

            var filepath = outputPath + "\\";

            if (_allAuthorsPosts)
                filepath += post.AuthorName;
            else
                filepath += post.SubredditName + "\\" + name;

            var filename = filepath + $"\\{name}_{post.SubredditName}_{post.Id}";

            var extention = GetExtention(url);

            if (!string.IsNullOrEmpty(extention))
            {
                OutputLine($"\tAdding Link {url}", true);
                links.Add(new ImageLink(post, url, filename));
                return links;
            }

            string htmlString = await GetHtml(url, token);
            if (string.IsNullOrWhiteSpace(htmlString)) return links;

            var caroselAlbum = htmlString.Contains(@"data-layout=""h""");

            if (caroselAlbum)
            {
                htmlString = await GetHtml(url + "/all", token);
                if (string.IsNullOrWhiteSpace(htmlString)) return links;
            }

            var gridAlbum = htmlString.Contains(@"data-layout=""g""");

            if (caroselAlbum && !gridAlbum) return links;

            var regPattern = new Regex(@"<img[^>]*?src\s*=\s*[""']?([^'"" >]+?)[ '""][^>]*?>", RegexOptions.IgnoreCase);

            var matchImageLinks = regPattern.Matches(htmlString);

            OutputLine($"\tFound {matchImageLinks.Count} image(s) from link.", true);

            foreach (Match m in matchImageLinks)
            {
                token.ThrowIfCancellationRequested();
                var imgurl = m.Groups[1].Value.Replace(".gifv", ".gif");

                if (!imgurl.Contains("imgur.com")) continue;

                if (gridAlbum)
                    imgurl = imgurl.Remove(imgurl.LastIndexOf('.') - 1, 1);
                var domain = new Uri(imgurl).DnsSafeHost;
                imgurl = imgurl.StartsWith("http://") ? imgurl : "http://" + imgurl.Substring(imgurl.IndexOf(domain, StringComparison.Ordinal));

                links.Add(new ImageLink(post, imgurl, filename));
            }

            return links;
        }
Beispiel #9
0
        /// <summary>
        /// Add an item to the hierarchy based on the item path
        /// </summary>
        /// <param name="item">Item to add</param>
        /// <returns>Added node</returns>
        protected virtual HierarchyNode AddIndependentFileNode(MSBuild.ProjectItem item, IDictionary<string, HierarchyNode> parentNodeCache)
        {
            if (item == null)
                throw new ArgumentNullException("item");
            if (parentNodeCache == null)
                throw new ArgumentNullException("parentNodeCache");
            //Contract.Ensures(Contract.Result<HierarchyNode>() != null);

            // Make sure the item is within the project folder hierarchy. If not, link it.
            string linkPath = item.GetMetadataValue(ProjectFileConstants.Link);
            if (String.IsNullOrEmpty(linkPath))
            {
                string projectFolder = new Uri(this.ProjectFolder).LocalPath;
                string itemPath = new Uri(Path.Combine(this.ProjectFolder, item.EvaluatedInclude)).LocalPath;
                if (!itemPath.StartsWith(projectFolder, StringComparison.OrdinalIgnoreCase))
                {
                    linkPath = Path.GetFileName(item.EvaluatedInclude);
                    item.SetMetadataValue(ProjectFileConstants.Link, linkPath);
                }
            }

            HierarchyNode currentParent = GetItemParentNode(item, parentNodeCache);
            return AddFileNodeToNode(item, currentParent);
        }
        /// <summary>
        /// Add an item to the hierarchy based on the item path
        /// </summary>
        /// <param name="item">Item to add</param>
        /// <returns>Added node</returns>
        private HierarchyNode AddIndependentFileNode(Microsoft.Build.Evaluation.ProjectItem item)
        {
            bool shouldLink = false;
            // Remove any redundant "." directories from the Include path.
            string includePath = item.Xml.Include.Replace("/./", "/").Replace("\\.\\", "\\");
            while (includePath.StartsWith("./", StringComparison.Ordinal) || includePath.StartsWith(".\\", StringComparison.Ordinal))
            {
                includePath = includePath.Substring(2);
            }

            // Don't set the property unless necessary to avoid dirtying the project.
            if (includePath != item.Xml.Include)
            {
                item.Xml.Include = includePath;
            }

            // Make sure the item is within the project folder hierarchy. If not, link it.
            string linkPath = item.GetMetadataValue(ProjectFileConstants.Link);
            if (String.IsNullOrEmpty(linkPath))
            {
                string projectFolder = new Uri(this.ProjectFolder).LocalPath;
                string itemPath = new Uri(Path.Combine(this.ProjectFolder, item.Xml.Include)).LocalPath;
                if (!itemPath.StartsWith(projectFolder, StringComparison.OrdinalIgnoreCase))
                {
                    shouldLink = true;
                }
            }

            HierarchyNode currentParent;
            // If the file is outside of the project dir, link to it and place
            // it under the project node. Do not attempt to create a '..'
            // folder node.
            if (shouldLink)
            {
                currentParent = this;
            }
            else
            {
                currentParent = GetItemParentNode(item);
            }

            HierarchyNode newNode = AddFileNodeToNode(item, currentParent);

            if (shouldLink)
            {
                linkPath = Path.GetFileName(item.Xml.Include);
                newNode.ItemNode.SetMetadata(ProjectFileConstants.Link, linkPath);
            }

            return newNode;
        }
Beispiel #11
0
		private static List<VideoInfo> getVideos(HtmlDocument doc, Uri baseUri)
		{
			List<VideoInfo> result = new List<VideoInfo>();

            var contentDiv = doc.DocumentNode.Descendants("div").FirstOrDefault(d => d.GetAttributeValue("id", "") == "content");
            var localWebPath = baseUri.AbsolutePath.Substring(0, baseUri.AbsolutePath.LastIndexOf("/") + 1);

            // when no links with data-extension attribute were found, use all that are below current path (daheim und unterwegs)
            var dataLinks = contentDiv.Descendants("a")
                .Where(a => a.GetAttributeValue("data-extension", "") != "" ||
                    (a.GetAttributeValue("href", "").StartsWith(localWebPath) && a.GetAttributeValue("href", "") != baseUri.AbsoluteUri && !a.GetAttributeValue("href", "").EndsWith("index.html") && !a.Descendants("strong").Any(s=>s.InnerText.Trim()=="mehr")))
                .GroupBy(a => a.GetAttributeValue("href", "")).ToDictionary(g => g.Key, g => g.ToList());

            foreach (var dataLink in dataLinks)
            {
                HtmlNode parent = null;

                if (dataLink.Value.Count > 1)
                {
                    parent = CommonParent(dataLink.Value[0], dataLink.Value[1], 3);
                }

                if (parent == null)
                    parent = dataLink.Value.First().ParentNode;

                var link = new Uri(baseUri, dataLink.Key).AbsoluteUri;
                if (!link.StartsWith("http")) continue;

                var imgNode = parent.Descendants("img").FirstOrDefault(img => img.GetAttributeValue("src", "") != "");
                var thumb = (imgNode != null) ? new Uri(baseUri, imgNode.GetAttributeValue("src", "")).AbsoluteUri : null;

                var teaserParagraph = parent.Descendants("p").FirstOrDefault(p => p.GetAttributeValue("class", "") == "teasertext");
                var desc = (teaserParagraph != null) ? HttpUtility.HtmlDecode(teaserParagraph.FirstChild.InnerText).Trim().Trim('|').Trim() : null;

                var durationDiv = parent.Descendants("div").FirstOrDefault(p => p.GetAttributeValue("class", "") == "duration");
                var length = (durationDiv != null) ? durationDiv.InnerText.Trim() : null;

                var pi = parent.Descendants("p").FirstOrDefault(p => p.GetAttributeValue("class", "") == "programInfo");
                var airdate = (pi != null) ? pi.FirstChild.InnerText.Replace('|', ' ').Trim() : null;
                if (airdate == null)
                {
                    var airdateTextNode = parent.Descendants("#text").FirstOrDefault(t => Regex.IsMatch(t.InnerText.Trim(), @"\d{2}\.\d{2}\.\d{4}"));
                    if (airdateTextNode != null)
                        airdate = airdateTextNode.InnerText.Trim();
                }

                var header = parent.Descendants().FirstOrDefault(h => h.Name == "h3" || h.Name == "h4");
                var title = (header != null) ? HttpUtility.HtmlDecode(header.InnerText.Replace("Video:", "").Trim()) : null;
                if (title == null)
                {
                    if (parent.Name == "li")
                    {
                        title = parent.Descendants("span").FirstOrDefault().InnerText;
                    }
                }
                result.Add(new VideoInfo { Title = title, Description = desc, Thumb = thumb, VideoUrl = link, Airdate = airdate, Length = length });
            }

			return result;
		}
 private bool IsExternalLink(string url)
 {
     try
     {
         var link = new Uri(url).Host;
         link = link.Replace("www.", string.Empty);
         return !link.StartsWith(_hostOfOriginalUrl);
     }
     catch (FormatException)
     {
         return false;
     }
 }
        private void RewriteImageSource( XElement root )
        {
            if( Location == null )
            {
                return;
            }

            if( root.Name == XName.Get( "Image", "http://schemas.microsoft.com/winfx/2006/xaml/presentation" ) )
            {
                var source = new Uri( root.Attribute( "Source" ).Value ).LocalPath;
                if( source.StartsWith( Location, StringComparison.OrdinalIgnoreCase ) )
                {
                    source = source.Substring( Location.Length );
                    if( source.StartsWith( "\\" ) )
                    {
                        source = source.Substring( 1 );
                    }
                    root.Attribute( "Source" ).Value = source;
                }
            }

            foreach( var child in root.Elements() )
            {
                RewriteImageSource( child );
            }
        }
Beispiel #14
0
        /// <summary>
        /// Initializes the name of the running service.
        /// </summary>
        private static void InitializeRunningServiceName()
        {
            try
            {
                // Usually, the application run into IIS, so HttpContext is initialized.
                HttpContext context = HttpContext.Current;
                if (context != null && context.Request != null)
                {
                    RunningServiceName = context.Request.ApplicationPath.Trim('/');
                }
                else
                {
                    string path = new System.Uri(typeof(ServiceConfiguration).Assembly.CodeBase).LocalPath;

                    // In case that we run into the debugger of Visual studio, web site might be located into a temporary folder.
                    string tempAspNetFiles = Path.Combine(Path.GetTempPath(), "Temporary ASP.NET Files" + Path.DirectorySeparatorChar);

                    if (path.StartsWith(tempAspNetFiles, StringComparison.OrdinalIgnoreCase))
                    {
                        path = path.Substring(tempAspNetFiles.Length);

                        char firstChar  = path[0];
                        int  foundIndex = path.IndexOfAny(new char[] { System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar }, 0);

                        if (foundIndex > 0)
                        {
                            path = path.Substring(0, foundIndex);
                        }
                    }
                    else
                    {
                        // Run in the context of automated tests.
                        string[] pathSplitted = path.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
                        int      index        = pathSplitted.Length;
                        if (index > 1)
                        {
                            index -= 2;
                            if (index > 1 && (string.Equals(pathSplitted[index], "Debug", StringComparison.OrdinalIgnoreCase) ||
                                              string.Equals(pathSplitted[index], "Release", StringComparison.OrdinalIgnoreCase)))
                            {
                                index--;
                            }
                            if (index > 1 && string.Equals(pathSplitted[index], "bin", StringComparison.OrdinalIgnoreCase))
                            {
                                index--;
                            }
                        }
                        else
                        {
                            index = 0;
                        }

                        path = pathSplitted[index];
                    }

                    if (!string.IsNullOrEmpty(path))
                    {
                        RunningServiceName = path;
                    }
                    else
                    {
                        RunningServiceName = "PIS.Ground.Core";
                    }
                }
            }
            catch
            {
                RunningServiceName = "PIS.Ground.Core";
            }
        }