コード例 #1
0
ファイル: PatternAtRule.cs プロジェクト: jakeranderson/Yate
        public void Render(CQ html, IYateDataContext dataContext)
        {
            var selector = _selector.GetValue(dataContext).ToString();
            var patterns = html.Select(selector);

            var patternHtml = CQ.CreateFragment(patterns.First());

            dataContext.PatternContainer.Add(
                _name.GetValue(dataContext).ToString(),
                new YatePattern(patternHtml, AtRules, RuleSets));

            html.Select(selector).Remove();
        }
コード例 #2
0
 public void Render(CQ dom)
 {
     foreach (var selector in _renderingParameters.Selectors)
     {
         ////todo: figure out prepend and such
         //if (renderingParameters.Expressions.Count() > 1)
         //{
         //    var method = renderingParameters.Expressions[1].ToLower();
         //    if (method == "append")
         //    {
         //        dom.Select(selector).Text(dom.Select(selector).Text() + renderingParameters.Expressions.FirstOrDefault());
         //    }
         //    else if (method == "prepend")
         //    {
         //        dom.Select(selector).Text(renderingParameters.Expressions.FirstOrDefault() + dom.Select(selector).Text());
         //    }
         //    else
         //    {
         //        dom.Select(selector).Text(renderingParameters.Expressions.FirstOrDefault());
         //    }
         //}
         //else
         //{
         dom.Select(selector).Text(_renderingParameters.Expressions.FirstOrDefault());
         //}
     }
 }
コード例 #3
0
        public void Render(CQ dom, string selector, IYateDataContext dataContext)
        {
            string format;

            if (_format != null)
            {
                format = _format.GetValue(dataContext).ToString();
            }
            else
            {
                format = dom.Select(selector).Text();
            }

            dom.Select(selector).Text(
                string.Format(format, _text.GetValue(dataContext)));
        }
コード例 #4
0
 static string[] ParseCommandsNames(CQ dom, IDomObject @event)
 {
     var commands =
         dom.Select(
             "td.today-name > span.command > div.today-member-name, td.name > span.command div.member-name",
             @event).Map(node => node.Cq().Text().Trim().Replace(Nbsp, ' ')).ToArray();
     Debug.Assert(commands.Count() == 2, "Teams names have not been recognized: " + @event.InnerHTML);
     return commands;
 }
コード例 #5
0
        static DateTimeOffset ParseEventDate(CQ dom, IDomObject @event)
        {
            var date = dom.Select("td.date", @event).Text().Trim();

            DateTimeOffset eventDate;
            DateTimeOffset.TryParse(date, out eventDate);

            return eventDate;
        }
コード例 #6
0
 static IEnumerable<ParsedEventData> ParseEvents(CQ dom, IDomObject @event)
 {
     return dom.Select("td.js-price", @event)
               .Map(node =>
                        {
                            var domNode = node.Cq();
                            var coefficient = double.Parse(domNode.Find("span.selection-link").Text().Trim());
                            var specification = domNode.Children().Remove().End().Text().Trim();
                            return new ParsedEventData {Coefficient = coefficient, Specification = specification};
                        });
 }
コード例 #7
0
        static IEnumerable<EventLine> ParseLines(CQ dom, IEnumerable<string> eventNames)
        {
            var events = dom.Select("table.foot-market tr.event-header");

            foreach (var @event in events)
            {
                var commands = ParseCommandsNames(dom, @event);

                var eventDate = ParseEventDate(dom, @event);

                var eventsData = ParseEvents(dom, @event).ToList();

                yield return
                    new EventLine(string.Join("-", commands), eventDate, CreateBets(eventNames, eventsData).ToArray());
            }
        }
コード例 #8
0
        public HTMLEnderecoParse(WebRequest requisicao, int cep)
        {
            var bytes = System.Text.Encoding.ASCII.GetBytes(string.Format("cepEntrada={0}&tipoCep=&cepTemp&metodo=buscarCep", cep));
            requisicao.ContentLength = bytes.Length;
            using (var os = requisicao.GetRequestStream())
            {
                os.Write(bytes, 0, bytes.Length);
                os.Close();
            }

            var resposta = requisicao.GetResponse();
            using (var responseStream = resposta.GetResponseStream())
            using (var reader = new StreamReader(responseStream))
            {
                var html = reader.ReadToEnd();
                _csQueryParsed = CQ.Create(html);
            }

            if (EhValido)
            {
                var htmlResp = _csQueryParsed.Select(".respostadestaque");
                Endereco = new Endereco
                {
                    Bairro = htmlResp.Eq(1).Contents().ToHtmlString().Trim()
                    ,
                    Cep = htmlResp.Eq(3).Contents().ToHtmlString().Trim()
                    ,
                    Cidade = htmlResp.Eq(2).Contents().ToHtmlString().Trim().Split('/')[0].Trim()
                    ,
                    Estado = htmlResp.Eq(2).Contents().ToHtmlString().Trim().Split('/')[1].Trim()
                    ,
                    TipoDeLogradouro = htmlResp.Eq(0).Contents().ToHtmlString().Trim().Split(' ')[0]
                };
                var logradouro = htmlResp.Eq(0).Contents().ToHtmlString().Trim().Split(' ');
                var logradouroCompleto = string.Empty;
                for (var i = 0; i < logradouro.Length; i++)
                {
                    if (i <= 0) continue;
                    if (logradouro[i] == "-") break;
                    logradouroCompleto += logradouro[i];
                    logradouroCompleto += " ";
                }
                Endereco.Logradouro = logradouroCompleto.Trim();
            }
        }
コード例 #9
0
        public void CheckChildren(string parentUrl, string childSelector, string html)
        {
            var dom = new CQ(html);
            var links = dom.Select(childSelector);
            var childrenChecked = 0;

            Parallel.ForEach(links, new ParallelOptions() { MaxDegreeOfParallelism = Config.MaxChildThreads }, o =>
            {
                childrenChecked++;
                var url = o.Attributes["href"];
                // ignore empty
                if (String.IsNullOrWhiteSpace(url))
                    return;
                // ignore javascript
                if (url.ToLower().StartsWith("javascript:"))
                    return;

                // jump out if url should be ignored
                if (Config.IgnoreUrlsStartingWith.Any(ignore => url.StartsWith(ignore)))
                    return;

                // check for relative url
                if (!url.StartsWith("http://") &&
                    !url.StartsWith("https://") &&
                    !url.StartsWith("//"))
                {
                    if (url.StartsWith("/"))
                        url = new Uri(parentUrl).GetLeftPart(UriPartial.Authority) + url;

                    else
                        url = parentUrl + url;
                }

                // check for protocol relative url
                if (url.StartsWith("//"))
                {
                    url = new Uri(parentUrl).GetLeftPart(UriPartial.Scheme) + url.Substring(2);
                }

                PerformCheck(url, null, true);
            });

            Console.WriteLine("Checked {0} children of {1}", childrenChecked, parentUrl);
        }
コード例 #10
0
        public static Dictionary<string, string> GetInputFields(CQ dom)
        {
            Dictionary<string, string> result = new Dictionary<string, string>();
            foreach (var v in dom.Find("input"))
            {
                var value = v.Cq().Attr("value");
                var key = v.Cq().Attr("name");
                if (!string.IsNullOrWhiteSpace(value) && !string.IsNullOrWhiteSpace(key))
                {
                    if (!result.ContainsKey(key))
                    {
                        result.Add(key, value);
                    }
                    else
                    {
                        result[key] = value;
                    }
                }
            }

            // Get all selects
            foreach (var s in dom.Select("select"))
            {
                var select = s.Cq();
                var key = select.Attr("name");
                foreach (var option in select.Children("option"))
                {
                    var opt = option.Cq();
                    if (!string.IsNullOrWhiteSpace(opt.Attr("selected")))
                    {
                        if (!result.ContainsKey(key))
                        {
                            result.Add(key, opt.Val());
                        }
                        else
                        {
                            result[key] = opt.Val();
                        }
                    }
                }
            }

            return result;
        }
コード例 #11
0
        public string Sort()
        {
            if (string.IsNullOrWhiteSpace(_content))
                return string.Empty;

            var dom = new CQ(_content);
            var root = dom.Select("Project");

            foreach (var element in root.Children())
            {
                if (element.RequiresSort())
                {
                    element.SortChildren();
                }

            }

            return dom.Render();
        }
コード例 #12
0
        public void ProcessDisplayElement(CQ parsedContent)
        {
            CQ elements = parsedContent.Select(ReplaceSelector);
            bool foundFirstScript = false;

            ////WE can minify the inline javascript, however this costs more than the time it saves from the bytes saved not sent over the network
            foreach (IDomObject element in elements)
            {
                string path = element.GetAttribute("src") ?? string.Empty;

                if (!path.EndsWith(BundlePath))
                {
                    if (PartOfBundle(path))
                    {
                        foundFirstScript = UpdateJavascriptPath(foundFirstScript, element);
                    }
                }
            }
        }
コード例 #13
0
        public void Render(CQ html, string selector, IYateDataContext dataContext)
        {
            var domObjects = html.Select(selector);
            var pattern = dataContext.PatternContainer[_patternName.GetValue(dataContext).ToString()];
            var model = _model.GetValue(dataContext);

            if (model is ICollection)
            {
                var enumModels = model as ICollection;

                foreach (var enumModel in enumModels)
                {
                    RunPattern(dataContext, enumModel, domObjects, pattern);
                }
            }
            else
            {
                RunPattern(dataContext, model, domObjects, pattern);
            }
        }
コード例 #14
0
        public void Process(CQ parsedContent)
        {
            if (_microDataEntries != null)
            {
                try
                {
                    var body = parsedContent.Select("body");
                    Dictionary<Guid, IContent> contentLookup = new Dictionary<Guid, IContent>();

                    foreach (MicroDataEntry semanticEntry in _microDataEntries)
                    {
                        Guid semanticType = semanticEntry.ContentType.GetValueOrDefault(Guid.Empty);

                        if (semanticType == Guid.Empty)
                        {
                            ProcessSymanticEntry(semanticEntry, body);
                        }
                        else if (WebContextualContentTypes.ContainsKey(semanticType))
                        {
                            if (!contentLookup.ContainsKey(semanticType))
                            {
                                contentLookup.Add(semanticType, WebContextualContentTypes[semanticType].GetCurrentContent(new WebContext()));
                            }

                            if (contentLookup[semanticType] != null)
                            {
                                ProcessSymanticEntry(semanticEntry, body);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    new Telligent.Evolution.Components.CSException(Telligent.Evolution.Components.CSExceptionType.UnknownError, "MicroDataPlugin unkonw error", ex).Log();
                }
            }
        }
コード例 #15
0
 private void ProcessSymanticEntry(MicroDataEntry semanticEntry, CQ body)
 {
     if (semanticEntry.Type == MicroDataType.itemprop ||
         semanticEntry.Type == MicroDataType.rel)
     {
         body.Select(semanticEntry.Selector).Attr(semanticEntry.Type.ToString(), semanticEntry.Value);
     }
     else if (semanticEntry.Type == MicroDataType.itemscope)
     {
         body.Select(semanticEntry.Selector).Each(node => node.SetAttribute("itemscope"));
         body.Select(semanticEntry.Selector).Attr("itemtype", semanticEntry.Value);
     }
     else
     {
         body.Select(semanticEntry.Selector).Attr(semanticEntry.Value, semanticEntry.Value);
     }
 }
コード例 #16
0
ファイル: LoginJob.cs プロジェクト: KRSogaard/UkData.Scraper
 public static SerializableDictionary<string, SerializableList<string>> GetDDFields(CQ dom)
 {
     var data = new SerializableDictionary<string, SerializableList<string>>();
     foreach (var dd in dom.Select(x => x.Cq()))
     {
         var topField = dd[0].InnerText.Trim();
         if (!data.ContainsKey(topField))
         {
             data.Add(ToolsHelper.MakeXmlSafe(topField), new SerializableList<string>());
         }
         if (!dd.Find("li").Any())
         {
             data[topField].Add(ToolsHelper.MakeXmlSafe(topField));
         }
         else
         {
             foreach (var li in dd.Find("li").Select(x => x.Cq()))
             {
                 data[topField].Add(li.Text().Trim());
             }
         }
     }
     return data;
 }
コード例 #17
0
        private void HandleInlineScripts(CQ parsedContent, ContentFragmentPageControl contentFragmentPage)
        {
            if (Configuration.OptomizeGlobalJs)
            {
                //Get the themes CSS files
                CQ elements = parsedContent.Select(BuildSelector);

                foreach (IDomObject element in elements)
                {
                    string src = element.GetAttribute("src");

                    if (!string.IsNullOrEmpty(src))
                    {
                        IBundledFile file = _bundledFileFactory.GetBundleFile("js", src, contentFragmentPage, "");

                        if (file != null)
                            Include(file);
                    }
                }
                //Get the themes axd files
                elements = parsedContent.Select(BuildAxdSelector);
                int i = 0;
                foreach (IDomObject element in elements)
                {
                    string src = element.GetAttribute("src");

                    if (!string.IsNullOrEmpty(src))
                    {
                        IBundledFile file = _bundledFileFactory.GetBundleFile("js", src, contentFragmentPage, i.ToString());

                        if (file != null)
                           Include(file);

                        i++;
                    }
                }
            }
        }
コード例 #18
0
        IEnumerable<string> ParseEventsNamesForEvent(CQ dom)
        {
            var tablebets = dom.Select("table.foot-market tbody:first", dom);

            var betsKind1 =
                dom.Select("th.coupone > div.markets-hint", tablebets)
                   .Map(node => node.Cq().Text().Trim().Replace(Nbsp, ValidSpace));
            var betsKind2 = dom.Select("th.coupone > span", tablebets).Map(node => node.Cq().Text());

            return betsKind1.Union(betsKind2).ToList();
        }
コード例 #19
0
ファイル: HtmlProperty.cs プロジェクト: jakeranderson/Yate
 public void Render(CQ dom, string selector, IYateDataContext dataContext)
 {
     //todo:figure out append, prepend, etc
     dom.Select(selector).Html(_htmlValue.GetValue(dataContext).ToString());
 }
コード例 #20
0
        public void TestInitiateAuthorizationGet()
        {
            var collection = new CollectionState(new Uri(SANDBOX_URI));
            Link tokenLink = collection.GetLink(Rel.OAUTH2_AUTHORIZE);
            IDictionary<String, String> formData = new Dictionary<String, String>();
            formData.Add("response_type", "code");
            formData.Add("client_id", "ABCD-EFGH-JKLM-NOPQ-RSTU-VWXY-0123-4567");
            formData.Add("redirect_uri", "https://familysearch.org/developers/sandbox-oauth2-redirect");
            IRestRequest request = new RedirectableRestRequest()
                .SetEntity(formData)
                .Build(tokenLink.Href, Method.GET);
            IRestResponse response = collection.Client.Handle(request);

            Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
            var cq = new CQ(response.Content);
            Assert.AreEqual(1, cq.Select("input#userName").Length);
            Assert.AreEqual(1, cq.Select("input#password").Length);
        }
コード例 #21
0
ファイル: TextProperty.cs プロジェクト: jakeranderson/Yate
 public override void Render(CQ dom, string selector, IYateDataContext dataContext)
 {
     //todo:figure out append, prepend, etc
     dom.Select(selector).Text(Values[0].GetValue(dataContext).ToString());
 }
コード例 #22
0
 public void Render(CQ dom, string selector, IYateDataContext dataContext)
 {
     //todo:figure out append, prepend, etc
     dom.Select(selector).Attr(_attributeName.GetValue(dataContext).ToString(), _attributeValue.GetValue(dataContext).ToString());
 }
コード例 #23
0
ファイル: HtmlComparer.cs プロジェクト: joeriks/DryHtml
        bool isExcludedSelectors(IDomObject node)
        {
            if (excludeSelectors == null) return false;

            var cq = new CQ(compareHtml).Select(selector);
            
            foreach (var excludeSelector in excludeSelectors)
            {
                var selExclude = cq.Select(excludeSelector);
                if (selExclude.Any(n => n.NodeName == node.NodeName && n.NodePath.SequenceEqual(node.NodePath) && n.Attributes.SequenceEqual(node.Attributes))) return true;
                
            }
            return false;
        }
コード例 #24
0
        public void TestInitiateAuthorizationInvalidParameter()
        {
            var collection = new CollectionState(new Uri(SANDBOX_URI));
            Link tokenLink = collection.GetLink(Rel.OAUTH2_AUTHORIZE);
            IDictionary<String, String> formData = new Dictionary<String, String>();
            formData.Add("response_type", "authorize_me");
            formData.Add("client_id", "ABCD-EFGH-JKLM-NOPQ-RSTU-VWXY-0123-4567");
            formData.Add("redirect_uri", "https://familysearch.org/developers/sandbox-oauth2-redirect");
            IRestRequest request = new RedirectableRestRequest()
                .ContentType(MediaTypes.APPLICATION_FORM_URLENCODED_TYPE)
                .SetEntity(formData)
                .Build(tokenLink.Href, Method.POST);
            IRestResponse response = collection.Client.Handle(request);

            Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
            var cq = new CQ(response.Content);
            Assert.AreEqual(1, cq.Select("p#error").Length);
        }
コード例 #25
0
ファイル: LoginJob.cs プロジェクト: KRSogaard/UkData.Scraper
 public static SerializableList<string> GetDDFieldsList(CQ dom)
 {
     var data = new SerializableList<string>();
     foreach (var dd in dom.Select(x => x.Cq()))
     {
         var topField = dd[0].InnerText.Trim();
         if (!data.Contains(topField))
         {
             data.Add(ToolsHelper.MakeXmlSafe(topField));
         }
     }
     return data;
 }
コード例 #26
0
        public void Process(CQ document)
        {
            if (_configuration.HasValue)
            {
                var pageContext = PublicApi.Url.ParsePageContext(HttpContext.Current.Request.Url.OriginalString);

                if (pageContext != null && pageContext.PageName == _pageName)
                {
                    if (_configuration.Value.RemoveHeader)
                    {
                        document.Select(".header-fragments").Remove();
                    }

                    if (_configuration.Value.RemoveFooter)
                    {
                        document.Select(".footer-fragments").Remove();
                    }
                }
            }
        }
コード例 #27
0
ファイル: clsHttp.cs プロジェクト: ramonliu/drama-tracer
 private void ParserYYets(CQ doc)
 {
     if (null != Links)
     {
         Links.Clear();
         Links = null;
     }
     Links = new List<DBEpisode>();
     string[] str = System.Web.HttpUtility.HtmlDecode(doc.Select("title")[0].InnerText).Split(',');
     this.Title = str[0] + "," + str[1];
     var items = doc.Select("ul[class=resod_list]");            
     foreach (IDomElement item in items)
     {
         var season = item.GetAttribute("season");
         if (season.CompareTo("103") == 0)
             season = "預告片區";
         else
             season = "第 " + season + " 季";
         foreach (IDomElement epi in item.ChildElements)
         {
             var format = epi.GetAttribute("format");
             var epiIdx = epi.GetAttribute("episode");
             var title = epi.ChildElements.ElementAt(0).ChildElements.ElementAt(1).ChildElements.ElementAt(0).GetAttribute("title");
             var downloads = epi.ChildElements.ElementAt(1).ChildElements.ElementAt(0);
             foreach (var download in downloads.ChildElements)
             {
                 if (download.NodeName.ToLower() == "a")
                 {
                     var href = download.GetAttribute("href");
                     if ((href == null) || (href.Length <= 0))
                         continue;
                     var kind = System.Web.HttpUtility.HtmlDecode(download.InnerText);
                     var dbepi = new DBEpisode();
                     dbepi.Format = format;
                     dbepi.Link = href;
                     dbepi.Name = title;
                     dbepi.Season = season;
                     dbepi.Kind = kind;
                     Links.Add(dbepi);
                 }
             }
         }
     }
 }
コード例 #28
-1
        public void BuildBundleData(ContentFragmentPageControl contentFragmentPage, CQ parsedContent)
        {
            if (Configuration.OptomizeGlobalCss)
            {
                //Get the themes CSS files
                CQ elements = parsedContent.Select(BuildSelector);

                foreach (IDomObject element in elements)
                {
                    string mediaType = element.GetAttribute("media") ?? "screen";

                    if (mediaType.IndexOf("screen", StringComparison.OrdinalIgnoreCase) > -1)
                    {
                        string src = element.GetAttribute("href");

                        if (!string.IsNullOrEmpty(src) && mediaType.IndexOf("dynamic-style.aspx", StringComparison.OrdinalIgnoreCase) < 0 && src.EndsWith(".css"))
                        {
                            IBundledFile file = _bundledFileFactory.GetBundleFile("css", src, contentFragmentPage, "");

                            if (file != null)
                            {
                                HandleLayoutCssPath(file);
                                
                                Include(file);
                            }
                        }
                    }
                }


            }
        }