public void Render(CQ html, IYateDataContext dataContext)
{
var selector = _selector.GetValue(dataContext).ToString();
var patterns = html.Select(selector);
var patternHtml = CQ.CreateFragment(patterns.First());
dataContext.PatternContainer.Add(
_name.GetValue(dataContext).ToString(),
new YatePattern(patternHtml, AtRules, RuleSets));
html.Select(selector).Remove();
}
public void Render(CQ dom)
{
foreach (var selector in _renderingParameters.Selectors)
{
////todo: figure out prepend and such
//if (renderingParameters.Expressions.Count() > 1)
//{
// var method = renderingParameters.Expressions[1].ToLower();
// if (method == "append")
// {
// dom.Select(selector).Text(dom.Select(selector).Text() + renderingParameters.Expressions.FirstOrDefault());
// }
// else if (method == "prepend")
// {
// dom.Select(selector).Text(renderingParameters.Expressions.FirstOrDefault() + dom.Select(selector).Text());
// }
// else
// {
// dom.Select(selector).Text(renderingParameters.Expressions.FirstOrDefault());
// }
//}
//else
//{
dom.Select(selector).Text(_renderingParameters.Expressions.FirstOrDefault());
//}
}
}
public void Render(CQ dom, string selector, IYateDataContext dataContext)
{
string format;
if (_format != null)
{
format = _format.GetValue(dataContext).ToString();
}
else
{
format = dom.Select(selector).Text();
}
dom.Select(selector).Text(
string.Format(format, _text.GetValue(dataContext)));
}
static string[] ParseCommandsNames(CQ dom, IDomObject @event)
{
var commands =
dom.Select(
"td.today-name > span.command > div.today-member-name, td.name > span.command div.member-name",
@event).Map(node => node.Cq().Text().Trim().Replace(Nbsp, ' ')).ToArray();
Debug.Assert(commands.Count() == 2, "Teams names have not been recognized: " + @event.InnerHTML);
return commands;
}
static DateTimeOffset ParseEventDate(CQ dom, IDomObject @event)
{
var date = dom.Select("td.date", @event).Text().Trim();
DateTimeOffset eventDate;
DateTimeOffset.TryParse(date, out eventDate);
return eventDate;
}
static IEnumerable<ParsedEventData> ParseEvents(CQ dom, IDomObject @event)
{
return dom.Select("td.js-price", @event)
.Map(node =>
{
var domNode = node.Cq();
var coefficient = double.Parse(domNode.Find("span.selection-link").Text().Trim());
var specification = domNode.Children().Remove().End().Text().Trim();
return new ParsedEventData {Coefficient = coefficient, Specification = specification};
});
}
static IEnumerable<EventLine> ParseLines(CQ dom, IEnumerable<string> eventNames)
{
var events = dom.Select("table.foot-market tr.event-header");
foreach (var @event in events)
{
var commands = ParseCommandsNames(dom, @event);
var eventDate = ParseEventDate(dom, @event);
var eventsData = ParseEvents(dom, @event).ToList();
yield return
new EventLine(string.Join("-", commands), eventDate, CreateBets(eventNames, eventsData).ToArray());
}
}
public HTMLEnderecoParse(WebRequest requisicao, int cep)
{
var bytes = System.Text.Encoding.ASCII.GetBytes(string.Format("cepEntrada={0}&tipoCep=&cepTemp&metodo=buscarCep", cep));
requisicao.ContentLength = bytes.Length;
using (var os = requisicao.GetRequestStream())
{
os.Write(bytes, 0, bytes.Length);
os.Close();
}
var resposta = requisicao.GetResponse();
using (var responseStream = resposta.GetResponseStream())
using (var reader = new StreamReader(responseStream))
{
var html = reader.ReadToEnd();
_csQueryParsed = CQ.Create(html);
}
if (EhValido)
{
var htmlResp = _csQueryParsed.Select(".respostadestaque");
Endereco = new Endereco
{
Bairro = htmlResp.Eq(1).Contents().ToHtmlString().Trim()
,
Cep = htmlResp.Eq(3).Contents().ToHtmlString().Trim()
,
Cidade = htmlResp.Eq(2).Contents().ToHtmlString().Trim().Split('/')[0].Trim()
,
Estado = htmlResp.Eq(2).Contents().ToHtmlString().Trim().Split('/')[1].Trim()
,
TipoDeLogradouro = htmlResp.Eq(0).Contents().ToHtmlString().Trim().Split(' ')[0]
};
var logradouro = htmlResp.Eq(0).Contents().ToHtmlString().Trim().Split(' ');
var logradouroCompleto = string.Empty;
for (var i = 0; i < logradouro.Length; i++)
{
if (i <= 0) continue;
if (logradouro[i] == "-") break;
logradouroCompleto += logradouro[i];
logradouroCompleto += " ";
}
Endereco.Logradouro = logradouroCompleto.Trim();
}
}
public void CheckChildren(string parentUrl, string childSelector, string html)
{
var dom = new CQ(html);
var links = dom.Select(childSelector);
var childrenChecked = 0;
Parallel.ForEach(links, new ParallelOptions() { MaxDegreeOfParallelism = Config.MaxChildThreads }, o =>
{
childrenChecked++;
var url = o.Attributes["href"];
// ignore empty
if (String.IsNullOrWhiteSpace(url))
return;
// ignore javascript
if (url.ToLower().StartsWith("javascript:"))
return;
// jump out if url should be ignored
if (Config.IgnoreUrlsStartingWith.Any(ignore => url.StartsWith(ignore)))
return;
// check for relative url
if (!url.StartsWith("http://") &&
!url.StartsWith("https://") &&
!url.StartsWith("//"))
{
if (url.StartsWith("/"))
url = new Uri(parentUrl).GetLeftPart(UriPartial.Authority) + url;
else
url = parentUrl + url;
}
// check for protocol relative url
if (url.StartsWith("//"))
{
url = new Uri(parentUrl).GetLeftPart(UriPartial.Scheme) + url.Substring(2);
}
PerformCheck(url, null, true);
});
Console.WriteLine("Checked {0} children of {1}", childrenChecked, parentUrl);
}
public static Dictionary<string, string> GetInputFields(CQ dom)
{
Dictionary<string, string> result = new Dictionary<string, string>();
foreach (var v in dom.Find("input"))
{
var value = v.Cq().Attr("value");
var key = v.Cq().Attr("name");
if (!string.IsNullOrWhiteSpace(value) && !string.IsNullOrWhiteSpace(key))
{
if (!result.ContainsKey(key))
{
result.Add(key, value);
}
else
{
result[key] = value;
}
}
}
// Get all selects
foreach (var s in dom.Select("select"))
{
var select = s.Cq();
var key = select.Attr("name");
foreach (var option in select.Children("option"))
{
var opt = option.Cq();
if (!string.IsNullOrWhiteSpace(opt.Attr("selected")))
{
if (!result.ContainsKey(key))
{
result.Add(key, opt.Val());
}
else
{
result[key] = opt.Val();
}
}
}
}
return result;
}
public string Sort()
{
if (string.IsNullOrWhiteSpace(_content))
return string.Empty;
var dom = new CQ(_content);
var root = dom.Select("Project");
foreach (var element in root.Children())
{
if (element.RequiresSort())
{
element.SortChildren();
}
}
return dom.Render();
}
public void ProcessDisplayElement(CQ parsedContent)
{
CQ elements = parsedContent.Select(ReplaceSelector);
bool foundFirstScript = false;
////WE can minify the inline javascript, however this costs more than the time it saves from the bytes saved not sent over the network
foreach (IDomObject element in elements)
{
string path = element.GetAttribute("src") ?? string.Empty;
if (!path.EndsWith(BundlePath))
{
if (PartOfBundle(path))
{
foundFirstScript = UpdateJavascriptPath(foundFirstScript, element);
}
}
}
}
public void Render(CQ html, string selector, IYateDataContext dataContext)
{
var domObjects = html.Select(selector);
var pattern = dataContext.PatternContainer[_patternName.GetValue(dataContext).ToString()];
var model = _model.GetValue(dataContext);
if (model is ICollection)
{
var enumModels = model as ICollection;
foreach (var enumModel in enumModels)
{
RunPattern(dataContext, enumModel, domObjects, pattern);
}
}
else
{
RunPattern(dataContext, model, domObjects, pattern);
}
}
public void Process(CQ parsedContent)
{
if (_microDataEntries != null)
{
try
{
var body = parsedContent.Select("body");
Dictionary<Guid, IContent> contentLookup = new Dictionary<Guid, IContent>();
foreach (MicroDataEntry semanticEntry in _microDataEntries)
{
Guid semanticType = semanticEntry.ContentType.GetValueOrDefault(Guid.Empty);
if (semanticType == Guid.Empty)
{
ProcessSymanticEntry(semanticEntry, body);
}
else if (WebContextualContentTypes.ContainsKey(semanticType))
{
if (!contentLookup.ContainsKey(semanticType))
{
contentLookup.Add(semanticType, WebContextualContentTypes[semanticType].GetCurrentContent(new WebContext()));
}
if (contentLookup[semanticType] != null)
{
ProcessSymanticEntry(semanticEntry, body);
}
}
}
}
catch (Exception ex)
{
new Telligent.Evolution.Components.CSException(Telligent.Evolution.Components.CSExceptionType.UnknownError, "MicroDataPlugin unkonw error", ex).Log();
}
}
}
private void ProcessSymanticEntry(MicroDataEntry semanticEntry, CQ body)
{
if (semanticEntry.Type == MicroDataType.itemprop ||
semanticEntry.Type == MicroDataType.rel)
{
body.Select(semanticEntry.Selector).Attr(semanticEntry.Type.ToString(), semanticEntry.Value);
}
else if (semanticEntry.Type == MicroDataType.itemscope)
{
body.Select(semanticEntry.Selector).Each(node => node.SetAttribute("itemscope"));
body.Select(semanticEntry.Selector).Attr("itemtype", semanticEntry.Value);
}
else
{
body.Select(semanticEntry.Selector).Attr(semanticEntry.Value, semanticEntry.Value);
}
}
public static SerializableDictionary<string, SerializableList<string>> GetDDFields(CQ dom)
{
var data = new SerializableDictionary<string, SerializableList<string>>();
foreach (var dd in dom.Select(x => x.Cq()))
{
var topField = dd[0].InnerText.Trim();
if (!data.ContainsKey(topField))
{
data.Add(ToolsHelper.MakeXmlSafe(topField), new SerializableList<string>());
}
if (!dd.Find("li").Any())
{
data[topField].Add(ToolsHelper.MakeXmlSafe(topField));
}
else
{
foreach (var li in dd.Find("li").Select(x => x.Cq()))
{
data[topField].Add(li.Text().Trim());
}
}
}
return data;
}
private void HandleInlineScripts(CQ parsedContent, ContentFragmentPageControl contentFragmentPage)
{
if (Configuration.OptomizeGlobalJs)
{
//Get the themes CSS files
CQ elements = parsedContent.Select(BuildSelector);
foreach (IDomObject element in elements)
{
string src = element.GetAttribute("src");
if (!string.IsNullOrEmpty(src))
{
IBundledFile file = _bundledFileFactory.GetBundleFile("js", src, contentFragmentPage, "");
if (file != null)
Include(file);
}
}
//Get the themes axd files
elements = parsedContent.Select(BuildAxdSelector);
int i = 0;
foreach (IDomObject element in elements)
{
string src = element.GetAttribute("src");
if (!string.IsNullOrEmpty(src))
{
IBundledFile file = _bundledFileFactory.GetBundleFile("js", src, contentFragmentPage, i.ToString());
if (file != null)
Include(file);
i++;
}
}
}
}
IEnumerable<string> ParseEventsNamesForEvent(CQ dom)
{
var tablebets = dom.Select("table.foot-market tbody:first", dom);
var betsKind1 =
dom.Select("th.coupone > div.markets-hint", tablebets)
.Map(node => node.Cq().Text().Trim().Replace(Nbsp, ValidSpace));
var betsKind2 = dom.Select("th.coupone > span", tablebets).Map(node => node.Cq().Text());
return betsKind1.Union(betsKind2).ToList();
}
public void Render(CQ dom, string selector, IYateDataContext dataContext)
{
//todo:figure out append, prepend, etc
dom.Select(selector).Html(_htmlValue.GetValue(dataContext).ToString());
}
public void TestInitiateAuthorizationGet()
{
var collection = new CollectionState(new Uri(SANDBOX_URI));
Link tokenLink = collection.GetLink(Rel.OAUTH2_AUTHORIZE);
IDictionary<String, String> formData = new Dictionary<String, String>();
formData.Add("response_type", "code");
formData.Add("client_id", "ABCD-EFGH-JKLM-NOPQ-RSTU-VWXY-0123-4567");
formData.Add("redirect_uri", "https://familysearch.org/developers/sandbox-oauth2-redirect");
IRestRequest request = new RedirectableRestRequest()
.SetEntity(formData)
.Build(tokenLink.Href, Method.GET);
IRestResponse response = collection.Client.Handle(request);
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
var cq = new CQ(response.Content);
Assert.AreEqual(1, cq.Select("input#userName").Length);
Assert.AreEqual(1, cq.Select("input#password").Length);
}
public override void Render(CQ dom, string selector, IYateDataContext dataContext)
{
//todo:figure out append, prepend, etc
dom.Select(selector).Text(Values[0].GetValue(dataContext).ToString());
}
public void Render(CQ dom, string selector, IYateDataContext dataContext)
{
//todo:figure out append, prepend, etc
dom.Select(selector).Attr(_attributeName.GetValue(dataContext).ToString(), _attributeValue.GetValue(dataContext).ToString());
}
bool isExcludedSelectors(IDomObject node)
{
if (excludeSelectors == null) return false;
var cq = new CQ(compareHtml).Select(selector);
foreach (var excludeSelector in excludeSelectors)
{
var selExclude = cq.Select(excludeSelector);
if (selExclude.Any(n => n.NodeName == node.NodeName && n.NodePath.SequenceEqual(node.NodePath) && n.Attributes.SequenceEqual(node.Attributes))) return true;
}
return false;
}
public void TestInitiateAuthorizationInvalidParameter()
{
var collection = new CollectionState(new Uri(SANDBOX_URI));
Link tokenLink = collection.GetLink(Rel.OAUTH2_AUTHORIZE);
IDictionary<String, String> formData = new Dictionary<String, String>();
formData.Add("response_type", "authorize_me");
formData.Add("client_id", "ABCD-EFGH-JKLM-NOPQ-RSTU-VWXY-0123-4567");
formData.Add("redirect_uri", "https://familysearch.org/developers/sandbox-oauth2-redirect");
IRestRequest request = new RedirectableRestRequest()
.ContentType(MediaTypes.APPLICATION_FORM_URLENCODED_TYPE)
.SetEntity(formData)
.Build(tokenLink.Href, Method.POST);
IRestResponse response = collection.Client.Handle(request);
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
var cq = new CQ(response.Content);
Assert.AreEqual(1, cq.Select("p#error").Length);
}
public static SerializableList<string> GetDDFieldsList(CQ dom)
{
var data = new SerializableList<string>();
foreach (var dd in dom.Select(x => x.Cq()))
{
var topField = dd[0].InnerText.Trim();
if (!data.Contains(topField))
{
data.Add(ToolsHelper.MakeXmlSafe(topField));
}
}
return data;
}
public void Process(CQ document)
{
if (_configuration.HasValue)
{
var pageContext = PublicApi.Url.ParsePageContext(HttpContext.Current.Request.Url.OriginalString);
if (pageContext != null && pageContext.PageName == _pageName)
{
if (_configuration.Value.RemoveHeader)
{
document.Select(".header-fragments").Remove();
}
if (_configuration.Value.RemoveFooter)
{
document.Select(".footer-fragments").Remove();
}
}
}
}
private void ParserYYets(CQ doc)
{
if (null != Links)
{
Links.Clear();
Links = null;
}
Links = new List<DBEpisode>();
string[] str = System.Web.HttpUtility.HtmlDecode(doc.Select("title")[0].InnerText).Split(',');
this.Title = str[0] + "," + str[1];
var items = doc.Select("ul[class=resod_list]");
foreach (IDomElement item in items)
{
var season = item.GetAttribute("season");
if (season.CompareTo("103") == 0)
season = "預告片區";
else
season = "第 " + season + " 季";
foreach (IDomElement epi in item.ChildElements)
{
var format = epi.GetAttribute("format");
var epiIdx = epi.GetAttribute("episode");
var title = epi.ChildElements.ElementAt(0).ChildElements.ElementAt(1).ChildElements.ElementAt(0).GetAttribute("title");
var downloads = epi.ChildElements.ElementAt(1).ChildElements.ElementAt(0);
foreach (var download in downloads.ChildElements)
{
if (download.NodeName.ToLower() == "a")
{
var href = download.GetAttribute("href");
if ((href == null) || (href.Length <= 0))
continue;
var kind = System.Web.HttpUtility.HtmlDecode(download.InnerText);
var dbepi = new DBEpisode();
dbepi.Format = format;
dbepi.Link = href;
dbepi.Name = title;
dbepi.Season = season;
dbepi.Kind = kind;
Links.Add(dbepi);
}
}
}
}
}
public void BuildBundleData(ContentFragmentPageControl contentFragmentPage, CQ parsedContent)
{
if (Configuration.OptomizeGlobalCss)
{
//Get the themes CSS files
CQ elements = parsedContent.Select(BuildSelector);
foreach (IDomObject element in elements)
{
string mediaType = element.GetAttribute("media") ?? "screen";
if (mediaType.IndexOf("screen", StringComparison.OrdinalIgnoreCase) > -1)
{
string src = element.GetAttribute("href");
if (!string.IsNullOrEmpty(src) && mediaType.IndexOf("dynamic-style.aspx", StringComparison.OrdinalIgnoreCase) < 0 && src.EndsWith(".css"))
{
IBundledFile file = _bundledFileFactory.GetBundleFile("css", src, contentFragmentPage, "");
if (file != null)
{
HandleLayoutCssPath(file);
Include(file);
}
}
}
}
}
}
