/// <summary>
/// Default constructor.
/// </summary>
/// <param name="html"></param>
public PiranhaHelper(WebPage parent, HtmlHelper html)
{
Parent = parent ;
Html = html ;
Gravatar = new GravatarHelper() ;
Culture = new CultureHelper() ;
}
public void TestWebPage()
{
var page = new WebPage<MyPage>("http://ya.ru", "<html><html>")
{
FuncParse = () => new MyPage {Id = 1, Title = "Test page"}
};
Assert.AreEqual(new MyPage {Id = 1, Title = "Test page"}, page.Model);
}
/// <summary>
/// Extract from the web page and return the results as a list.
/// </summary>
/// <param name="page">The web page to extract from.</param>
/// <returns>The results of the extraction as a List.</returns>
public IList<Object> ExtractList(WebPage page)
{
Listeners.Clear();
var listener = new ListExtractListener();
AddListener(listener);
Extract(page);
return listener.List;
}
public void Search(string website, string regexPattern)
{
List<string> output = new List<string>();
_regexPattern = regexPattern;
webpage = new WebPage();
webpage.WebPageLoaded += WebPageLoadedSearchCalled;
webpage.Load(website);
}
protected void Page_Load(object sender, EventArgs e)
{
webPage = this.Page as WebPage;
if (!IsPostBack)
{
ltlMenu.Text = webPage.SideMenus;
}
}
/// <summary>
/// Initializes a new instance of the WebPageDecorator class
/// </summary>
/// <param name="webPage">The web page model.</param>
/// <param name="context">The data context.</param>
public WebPageDecorator(WebPage webPage, IDataContext context)
{
if (webPage is WebPageDecorator)
Model = ((WebPageDecorator)webPage).Model;
else
Model = webPage;
this.Context = context;
Model.CopyTo(this, new string[] { "Widgets", "Roles", "Scripts", "StyleSheets" });
}
protected void Page_Load(object sender, EventArgs e)
{
if (this.Page is WebPage)
{
webPage = (this.Page as WebPage);
}
else
{
webPage = new WebPage(1);
}
}
protected void btnSave_Click(object sender, EventArgs e)
{
var page = new WebPage();
if (IsLoaded)
page.CopyPropertiesFrom(OriginalPage);
page.ParentPageId = null;
if (!String.IsNullOrEmpty(cboParentPages.SelectedValue))
page.ParentPageId = Convert.ToInt32(cboParentPages.SelectedValue);
page.Name = txtName.Text;
page.CompanyId = Company.CompanyId;
page.Description = txtDescription.Value.Replace("$0", "<br/>");
page.IsInMenu = chkIsInMenu.Checked;
page.IsPublished = chkIsPublished.Checked;
page.CanComment = chkCanComment.Checked;
page.MasterPage = cboMasterPage.Text;
if (page.IsPublished)
{
if (!page.PublishedDate.HasValue)
page.PublishedDate = DateTime.Now;
}
else
page.PublishedDate = null;
page.RedirectUrl = null;
if (!String.IsNullOrEmpty(txtRedirectUrl.Text))
page.RedirectUrl = txtRedirectUrl.Text;
page.ModifiedDate = DateTime.Now;
if (!page.UserId.HasValue)
page.UserId = User.Identity.UserId;
SiteManager.Save(page, txtTags.Text);
if (((WebControl)sender).ID == "btnSaveAndNew")
{
Response.Redirect("WebPage.aspx");
return;
}
//
//Close the modal popup and redirect for WebPages.aspx
//
ClientScript.RegisterStartupScript(this.GetType(), "close", "top.$.LightboxObject.close();", true);
//Response.Redirect("WebPages.aspx");
}
static void Main(string[] args)
{
WebPage webInfo = new WebPage("http://zhan.renren.com/flipped2012/more?page=1&count=20");
//"http://zhan.renren.com/ranzhi12/more?page=1&count=20"
string context = webInfo.Context;
string html = webInfo.M_html;
List<Link> result = webInfo.Links;
IEnumerable<IGrouping<string,Link>> tmp = result.GroupBy(o => o.Tag);
//var tmp2 = result.Where(o => o.Tag == "img" & (o.Src.IndexOf("xlarge") > -1 || o.Src.IndexOf("original") > -1)).ToList();
var imgs = result.Where(o => o.Tag == "img").ToList();
}
protected void Page_Load(object sender, EventArgs e)
{
if (this.Page is WebPage)
{
webPage = (this.Page as WebPage);
}
else
{
webPage = new WebPage(1);
}
Page.Title = webPage.WebName;
ltlKeyword.Text = "<meta name=\"keywords\" content=\"" + webPage.Keywords + "\"/>";
ltlDescription.Text = "<meta name=\"description\" content=\"" + webPage.Description + "\"/>";
}
public WebPageHttpHandler(string virtualPath)
{
// create a new WebPage instance for use when processing the request
this.Page = (WebPage)WebPageBase.CreateInstanceFromVirtualPath(virtualPath);
Func<WebPageRenderingBase> func = null;
if (func == null)
{
func = delegate
{
return StartPage.GetStartPage(this.Page, "_PageStart", System.Web.WebPages.WebPageHttpHandler.GetRegisteredExtensions());
};
}
this._startPage = new Lazy<WebPageRenderingBase>(func);
}
public static string RenderWebPage(WebPage page, StartPage startPage = null) {
var writer = new StringWriter();
// Create an actual dummy HttpContext that has a request object
var filename = "default.aspx";
var url = "http://localhost/default.aspx";
var request = new HttpRequest(filename, url, null);
var httpContext = new HttpContext(request, new HttpResponse(new StringWriter(new StringBuilder())));
var pageContext = Util.CreatePageContext(httpContext);
page.ExecutePageHierarchy(pageContext, writer, startPage);
return writer.ToString();
}
public static string RenderWebPage(WebPage page, StartPage startPage = null, HttpRequestBase request = null)
{
var writer = new StringWriter();
// Create an actual dummy HttpContext that has a request object
var filename = "default.aspx";
var url = "http://localhost/default.aspx";
request = request ?? CreateTestRequest(filename, url).Object;
var httpContext = CreateTestContext(request);
var pageContext = new WebPageContext { HttpContext = httpContext.Object };
page.ExecutePageHierarchy(pageContext, writer, startPage);
return writer.ToString();
}
private void WebPageLoadedSearchCalled(object sender, WebPage.WebPageArguments args)
{
string htmlData = webpage.htmlData;
if (htmlData != null)
{
System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(_regexPattern);
results = regex.Matches(htmlData);
}
if(InfoReady != null)
{
InfoReady(this, new InfoReadyEventArgs("info ready"));
}
// webpage.WebPageLoaded -= WebPageLoadedSearchCalled;
}
public static void InjectContext(WebPage razorWebPage, MacroModel macro, INode currentPage) {
var context = HttpContext.Current;
var contextWrapper = new HttpContextWrapper(context);
//inject http context - for request response
HttpContext.Current.Trace.Write("umbracoMacro", string.Format("Loading Macro Script Context (file: {0})", macro.Name));
razorWebPage.Context = contextWrapper;
HttpContext.Current.Trace.Write("umbracoMacro", string.Format("Done Loading Macro Script Context (file: {0})", macro.Name));
//Inject Macro Model And Parameters
if (razorWebPage is IMacroContext) {
HttpContext.Current.Trace.Write("umbracoMacro", string.Format("Boxing Macro Script MacroContext (file: {0})", macro.Name));
var razorMacro = (IMacroContext)razorWebPage;
HttpContext.Current.Trace.Write("umbracoMacro", string.Format("Done Boxing Macro Script MacroContext (file: {0})", macro.Name));
HttpContext.Current.Trace.Write("umbracoMacro", string.Format("Loading Macro Script Model (file: {0})", macro.Name));
razorMacro.SetMembers(macro, currentPage);
HttpContext.Current.Trace.Write("umbracoMacro", string.Format("Done Loading Macro Script Model (file: {0})", macro.Name));
}
}
void foo(string s)
{
WebPage p1 = new WebPage();
MyWebPage p2 = new MyWebPage();
p1.foo();
p2.foo();
a=3;b=4;c=5;
if (s > "xyz")
{
foo("hello");
}
if (s <= "xyz")
{
print("good");
print("bye");
}
if (x==y)
print("z");
}
/// <summary>
/// Fetches the page a URL refers to, and returns a WebPage struct.
/// </summary>
/// <param name="url">The URL to fetch.</param>
/// <returns>A WebPage struct containing the domain and the page itself.</returns>
public static WebPage FetchURL(string url)
{
try
{
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1";
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (Regex.IsMatch(response.ContentType, @"^(text/.*|application/((rss|atom|rdf)\+)?xml(;.*)?)$"))
{
Stream responseStream = response.GetResponseStream();
System.Text.Encoding encode = System.Text.Encoding.UTF8;
StreamReader stream = new StreamReader(responseStream, encode);
StringBuilder sb = new StringBuilder();
sb.Append(stream.ReadToEnd());
WebPage webPage = new WebPage();
webPage.Domain = response.ResponseUri.Scheme + "://" + response.ResponseUri.Host;
webPage.Page = sb.ToString();
response.Close();
stream.Close();
return webPage;
}
return new WebPage();
}
catch (System.Exception ex)
{
// Propagate exceptions upwards
throw ex;
}
}
public void ValidatePage(Page page, WebPage webPage)
{
ValidateTag(page, webPage, new Regex(DocPageValidatorResources.OwnerRegex),
DocPageValidatorResources.OwnerTagName,
DocPageValidatorResources.OwnerTagSelector,
DocPageValidatorResources.MultipleOwnerTagsErrorType,
DocPageValidatorResources.MultipleOwnerTagsErrorMessage,
DocPageValidatorResources.ImproperOwnerTagErrorType,
DocPageValidatorResources.ImproperOwnerTagErrorMessage);
ValidateTag(page, webPage, new Regex(DocPageValidatorResources.ReviewerRegex),
DocPageValidatorResources.ReviewerTagName,
DocPageValidatorResources.ReviewerTagSelector,
DocPageValidatorResources.MultipleReviewerTagsErrorType,
DocPageValidatorResources.MultipleReviewerTagsErrorMessage,
DocPageValidatorResources.ImproperReviewerTagErrorType,
DocPageValidatorResources.ImproperReviewerTagErrorMessage);
ValidateUpdatedOrReviewed(page, webPage, DocPageValidatorResources.UpdatedTagName);
ValidateUpdatedOrReviewed(page, webPage, DocPageValidatorResources.ReviewedTagName);
ValidateReviewedAfterUpdated(page);
ValidateReviewedAfterOldestAllowedDate(page);
ValidateTitle(page, webPage);
}
private static object[] CreateMethodParameters(WebPage page, ParameterInfo[] parameters)
{
var eventArgument = page.Request["__event_argument"];
var objects = new object[parameters.Length];
if (!string.IsNullOrWhiteSpace(eventArgument))
{
var argumentData = eventArgument.Split(',');
for (int i = 0; i < parameters.Length && i < argumentData.Length; i++)
{
objects[i] = Convert.ChangeType(argumentData[i], parameters[i].ParameterType);
}
}
else
{
for (int i = 0; i < parameters.Length; i++)
{
var name = parameters[i].Name;
if (page.Request[name] != null)
{
objects[i] = Convert.ChangeType(page.Request[name], parameters[i].ParameterType);
}
else if (name == "urlData" || parameters[i].ParameterType == typeof(string[]))
{
objects[i] = page.UrlData.ToArray();
}
else if (name == "request" || parameters[i].ParameterType == typeof(HttpRequest))
{
objects[i] = page.Request;
}
else if (parameters[i].ParameterType == typeof(ExpandoObject))
{
objects[i] = page.Request.Form.ToDynamic();
}
}
}
return objects;
}
public EnhancedHtmlHeading1(WebPage page, string selector) : base(page, selector) { }
private void ReadHttpResponse(HttpWebResponse response)
{
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
throw new NotFoundException(currentUrl);
case HttpStatusCode.Redirect:
throw new RedirectLoopException(GetRedirectUrl(response));
}
string body;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
body = reader.ReadToEnd();
}
if (response.StatusCode != HttpStatusCode.OK)
{
if (response.StatusCode == HttpStatusCode.InternalServerError)
{
string exceptionMessage = ParseStackTrace(body);
if (exceptionMessage != null) throw new AspServerException(exceptionMessage);
}
Console.WriteLine(body);
throw new BadStatusException(response.StatusCode);
}
currentPage = new WebPage(body);
}
public override bool ParseHtml(WebPage page, out RequestInfo requestInfo)
{
requestInfo = new RequestInfo();
return(true);
}
public ActionResult AddEmployee(Employee e, ImageFile photo, string webpage, string tmpl)
{
string dbPhotoPath = "";
if (photo != null)
{
photo.ValidateForUpload(true);
if (ModelState.IsValid)
{
if (photo.Save("emp", new System.Drawing.Size(Config.Employee.Photo.Width, Config.Employee.Photo.Height), false))
{
dbPhotoPath = "/" + photo.SavePath;
}
}
}
else
{
ModelState.AddModelError("photo", "Photo does not exist.");
}
if (!string.IsNullOrEmpty(dbPhotoPath))
{
if (ModelState.IsValid)
{
DBDataContext db = Utils.DB.GetContext();
WebPage page = db.WebPages.SingleOrDefault(x => x.ID == Convert.ToInt32(webpage));
if (page != null)
{
Employee emp = new Employee()
{
FirstName = e.FirstName,
LastName = e.LastName,
Title = e.Title,
Description = e.Description,
EmailAddress = e.EmailAddress,
Photo = dbPhotoPath
};
page.Employees.Add(emp);
try
{
db.SubmitChanges();
return(RedirectToAction("Employees", "Body", new { controller = "Body", action = "Employees", webpage = webpage, tmpl = tmpl }));
}
catch (Exception ex)
{
ModelState.AddModelError("", "An unknown error occurred. Please try again in few minutes.");
ErrorHandler.Report.Exception(ex, "Body/AddEmployee[HTTPPOST]");
}
}
else
{
ModelState.AddModelError("", "Web Page does not exist.");
}
}
}
if (System.IO.File.Exists(HttpContext.Server.MapPath(dbPhotoPath)))
{
System.IO.File.Delete(HttpContext.Server.MapPath(dbPhotoPath));
}
ViewData["WebPageID"] = webpage;
ViewData["Template"] = tmpl;
ViewData["Title"] = "Add Employee";
ViewData["Action"] = "AddEmployee";
return(View("ManageEmployee", e));
}
protected void Report()
{
string Html =
"</head>\n<body class='RptBody'>\n" +
WebPage.Table("align='center' width='100%'") +
WebPage.Tabs(1) + "<tr>\n" +
WebPage.Tabs(2) + "<td align='center'>\n" +
WebPage.Tabs(3) + WebPage.Div((rdoPrivate.Checked ? "Private" : "Corporate") + " Jobs", "center", "RptTitle") +
WebPage.Tabs(5) + WebPage.Div(Fmt.Dt(dtBeg) + " - " + Fmt.Dt(dtEnd), "center", "RptSubTitle") +
WebPage.Tabs(2) + "</td>\n" +
WebPage.Tabs(1) + "</tr>\n" +
WebPage.Tabs(1) + "<tr>\n" +
WebPage.Tabs(2) + WebPage.Space(1, 10) +
WebPage.Tabs(1) + "</tr>\n" +
WebPage.TableEnd();
Html +=
WebPage.Table("align='center'") +
WebPage.Tabs(1) + "<tr>\n" +
WebPage.Tabs(2) + "<td></td>\n" + // JobRno
WebPage.Tabs(2) + WebPage.Space(15, 1) +
WebPage.Tabs(2) + "<td></td>\n" + // JobDate
WebPage.Tabs(2) + WebPage.Space(15, 1) +
WebPage.Tabs(2) + "<td></td>\n" + // JobDesc
WebPage.Tabs(2) + WebPage.Space(15, 1) +
WebPage.Tabs(2) + "<td></td>\n" + // total servings
WebPage.Tabs(1) + "</tr>\n" +
WebPage.Tabs(1) + "<tr>\n" +
WebPage.Tabs(2) + "<th>Job #</th>\n" + // JobRno
WebPage.Tabs(2) + "<th></th>\n" +
WebPage.Tabs(2) + "<th>Date</th>\n" + // JobDate
WebPage.Tabs(2) + "<th></th>\n" +
WebPage.Tabs(2) + "<th>Description</th>\n" + // JobDesc
WebPage.Tabs(2) + "<th></th>\n" +
WebPage.Tabs(2) + "<th>Servings</th>\n" + // total servings
WebPage.Tabs(1) + "</tr>\n";
Response.Write(Html);
string JobDesc = "(Case When Len(j.JobDesc) <> 0 Then j.JobDesc Else Coalesce(u.Name, c.Name, '') + ' - ' + j.EventType End)";
string Sql = String.Format(
"Select j.JobRno, JobDate, " + JobDesc + " as JobDesc, " +
"(IsNull(j.NumMenServing, 0) + IsNull(j.NumWomenServing, 0) + IsNull(j.NumChildServing, 0)) As NumServings " +
"From mcJobs j " +
"Left Join Contacts c on j.ContactRno = c.ContactRno " +
"Left Join Customers u on c.CustomerRno = u.CustomerRno " +
"Where j.JobType = {0} And " +
"j.JobDate Between {1} And {2} And " +
"IsNull(j.ProposalFlg, 0) = 0 And " +
"j.CancelledDtTm Is Null " +
"Order By j.JobDate, j.JobRno, " + JobDesc,
DB.Put(rdoPrivate.Checked ? "Private" : "Corporate"),
DB.PutDtTm(dtBeg),
DB.PutDtTm(dtEnd));
try
{
DataTable dt = db.DataTable(Sql);
foreach (DataRow dr in dt.Rows)
{
int JobRno = DB.Int32(dr["JobRno"]);
string JobDate = Fmt.Dt(DB.DtTm(dr["JobDate"]));
string Desc = DB.Str(dr["JobDesc"]);
string Servings = Fmt.Num(DB.Int32(dr["NumServings"]));
Html =
WebPage.Tabs(1) + "<tr>\n" +
WebPage.Tabs(2) + "<td align='right'>" + JobRno + "</td>\n" +
WebPage.Tabs(2) + "<td></td>\n" +
WebPage.Tabs(2) + "<td>" + JobDate + "</td>\n" +
WebPage.Tabs(2) + "<td></td>\n" +
WebPage.Tabs(2) + "<td>" + Desc + "</td>\n" +
WebPage.Tabs(2) + "<td></td>\n" +
WebPage.Tabs(2) + "<td align='right'>" + Servings + "</td>\n" +
WebPage.Tabs(1) + "</tr>\n";
Response.Write(Html);
}
}
catch (Exception Ex)
{
Err Err = new Err(Ex, Sql);
Response.Write(Err.Html());
}
Html = WebPage.TableEnd();
Response.Write(Html);
Response.End();
}
public async Task <Company> AboutCompany(string ticker)
{
// тикер должен быть написан большими буквами, иначе возможны ошибки в запросах к другим сайтам
ticker = ticker.ToUpper();
string result = null;
string compName = "";
string sector = "";
int financialStrength = 0;
int profitabilityRank = 0;
int valuationRank = 0;
double price = 0;
string stockId = "";
WebRequest request = WebRequest.Create($"https://www.gurufocus.com/reader/_api/stocks/{ticker}/summary");
WebResponse response;
try
{
response = await request.GetResponseAsync();
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
result = reader.ReadToEnd();
}
}
response.Close();
using (JsonDocument doc = JsonDocument.Parse(result))
{
JsonElement root = doc.RootElement;
sector = root.GetProperty("sector").GetString();
compName = root.GetProperty("company").GetString();
financialStrength = root.GetProperty("rank_balancesheet").GetInt32();
profitabilityRank = root.GetProperty("rank_profitability").GetInt32();
stockId = root.GetProperty("stockid").GetString();
price = root.GetProperty("price").GetDouble();
}
valuationRank = GetValuationRank(ticker);
}
catch
{
return(null);
}
//грузим описание и лого с сайта Тинькофф
ScrapingBrowser browser = new ScrapingBrowser();
WebPage webPage = browser.NavigateToPage(new Uri($"https://www.tinkoff.ru/invest/stocks/{ticker}/"));
string compDescription = "";
var divs = webPage.Html.CssSelect("div");
var divInfo = divs.FirstOrDefault(x => x.Attributes["data-qa-file"]?.Value == "SecurityInfoPure");
if (divInfo != null)
{
// копируем описание компании и удаляем оттуда ненужную строку
compDescription = divInfo.InnerText.Replace("Официальный сайт компании", "");
}
// проверим есть ли лого компании на сервере, если нет - загрузим
string logoLink = "";
CloudinaryDotNet.Account cloudinaryAccount = new CloudinaryDotNet.Account(
_appSettings.CloudinaryCloudName,
_appSettings.CloudinaryApiKey,
_appSettings.CloudinaryApiSecret);
Cloudinary cloudinary = new Cloudinary(cloudinaryAccount);
var checkLogoInCloud = cloudinary.GetResource(ticker);
if (checkLogoInCloud.StatusCode != HttpStatusCode.OK)
{
var divLogo = webPage.Html.CssSelect(".Avatar-module__root_size_xl_1IN66");
var spanLogo = divLogo.CssSelect(".Avatar-module__image_2WFrC");
List <string> logoUrls = new List <string>();
foreach (var c in spanLogo)
{
logoUrls.Add(c.Attributes["style"]?.Value);
}
if (logoUrls?.Count != 0)
{
logoLink = logoUrls.First();
// заменяем ненужную часть ссылки на https и удаляем последний символ ")"
logoLink = logoLink.Replace("background-image:url(", "https:");
logoLink = logoLink.Remove(logoLink.Length - 1);
}
else
{
logoLink = $"https://storage.googleapis.com/iex/api/logos/{ticker}.png";
}
var uploadParams = new ImageUploadParams()
{
File = new FileDescription(ticker + ".png", logoLink),
PublicId = ticker
};
var uploadResult = cloudinary.Upload(uploadParams);
logoLink = uploadResult.SecureUrl.ToString();
}
else
{
logoLink = checkLogoInCloud.SecureUrl.ToString();
}
return(new Company
{
FinancialStrength = financialStrength,
ProfitabilityRank = profitabilityRank,
ValuationRank = valuationRank,
Sector = sector,
Name = compName,
Description = compDescription,
Ticker = ticker,
LogoUrl = logoLink,
StockId = stockId,
Price = price
});
}
public CountMetaTagCommand(WebPage page, List <string> metaTags) : base(page)
{
this.metaTags = metaTags;
}
public IEnumerable <Gym> RetrieveGyms(IEnumerable <string> urls)
{
List <Gym> results = new List <Gym>();
for (int i = 0; i < urls.Count(); i++)
{
string url = urls.ElementAt(i);
WebPage page = browser.NavigateToPage(new Uri(url));
HtmlNode content = page.Html.CssSelect(contentClass).First();
List <HtmlNode> properties = content.ChildNodes.ToList();
int index = 0;
Gym gym = new Gym()
{
Url = url
};
//Manually fetch properties
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Nombre:", StringComparison.InvariantCultureIgnoreCase)
);
gym.Name = FirstLetterToUpper(properties.ElementAt(index + 1).InnerText.Trim().ToLower());
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Domicilio:", StringComparison.InvariantCultureIgnoreCase)
);
gym.Address = properties.ElementAt(index + 1).InnerText.Replace("nº", "").Replace("º", "").Trim() + ", Madrid, Spain";
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Codigo postal:", StringComparison.InvariantCultureIgnoreCase)
);
gym.ZipCode = properties.ElementAt(index + 1).InnerText.Trim();
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Poblacion:", StringComparison.InvariantCultureIgnoreCase)
);
gym.City = properties.ElementAt(index + 1).InnerText.Trim();
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Horario:", StringComparison.InvariantCultureIgnoreCase)
);
gym.Hours = properties.ElementAt(index + 1).InnerText.Trim();
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Telefono:", StringComparison.InvariantCultureIgnoreCase)
);
gym.Phones = properties.ElementAt(index + 1).InnerText.Trim();
index = properties.FindIndex(n =>
n.Name.Equals("b", StringComparison.InvariantCultureIgnoreCase) &&
n.InnerText.Equals("Web:", StringComparison.InvariantCultureIgnoreCase)
);
gym.GymWeb = properties.ElementAt(index + 1).InnerText.Trim();
results.Add(gym);
Console.WriteLine("Processed Gym " + gym.Name + " (" + (i + 1) + " of " + urls.Count() + ")");
}
return(results);
}
public KeyValuePair <string, string> GetSideBarMenuItem(string item, WebPage page)
{
return(ExpandSideBarMenu(page).FirstOrDefault(m => string.Compare(m.Key, item, true) == 0));
}
public static Post AnalyzePost(string inPostUrl, IModelRepository inRepo, bool isOnFrontPage, bool inFetchCommentsVotes, ScrapingBrowser Browser = null)
{
Post newPost = new Post();
newPost.HrefLink = inPostUrl;
newPost.IsOnFrontPage = isOnFrontPage;
StringBuilder output = new StringBuilder();
output.AppendFormat("Post - {0,-90}", inPostUrl);
HtmlNode mainContent = null;
HtmlNode mainNode = null;
if (Browser != null)
{
WebPage PageResult = Browser.NavigateToPage(new Uri(inPostUrl));
mainContent = PageResult.Html.Descendants().SingleOrDefault(x => x.Id == "content-main");
mainNode = PageResult.Html;
//log.Info(mainNode.InnerHtml);
}
else
{
HtmlWeb htmlWeb = new HtmlWeb();
HtmlDocument htmlDocument = htmlWeb.Load(inPostUrl);
mainContent = htmlDocument.DocumentNode.Descendants().SingleOrDefault(x => x.Id == "content-main");
mainNode = htmlDocument.DocumentNode;
}
// first, Node ID
int nodeId;
string votesLink;
if (PostAnalyzer.ScrapePostID(mainContent, out nodeId, out votesLink))
{
newPost.Id = nodeId;
newPost.VotesLink = votesLink;
}
if (inRepo.PostAlreadyExists(newPost.Id)) // check for Post ID already in the repo
{
log.WarnFormat("WARNING - Post with ID {0} already in the database", newPost.Id);
return(null);
}
output.AppendFormat(" ID - {0,5}", newPost.Id);
// title
var titleHtml = mainContent.Descendants().Single(n => n.GetAttributeValue("class", "").Equals("node")).Descendants("h1").ToList();
newPost.Title = titleHtml[0].InnerText;
// text of the post
var postText = mainContent.Descendants().First(n => n.GetAttributeValue("class", "").Equals("node"));
if (postText != null)
{
int n1 = postText.InnerText.IndexOf("dodaj komentar");
newPost.Text = postText.InnerText.Substring(n1 + 20);
}
// date posted
newPost.DatePosted = ScrapePostDate(mainContent);
output.AppendFormat(" Date - {0}", newPost.DatePosted.ToString("dd/MM/yyy hh:mm"));
// author
string author, authorHtml;
PostAnalyzer.ScrapePostAuthor(mainNode, out author, out authorHtml);
output.AppendFormat(" Username - {0,-18}", author);
// check if user exists, add him if not
User user = inRepo.GetUserByName(author);
if (user == null)
{
user = new User {
Name = author, NameHtml = authorHtml
};
//Console.WriteLine(user.Name + " ; " + user.NameHtml);
inRepo.AddUser(user);
}
newPost.Author = user;
newPost.NumCommentsScrapped = CommentsAnalyzer.ScrapePostCommentsNum(mainContent);
if (newPost.NumCommentsScrapped < 0)
{
log.Error("ERROR - scrapping number of comments");
}
output.AppendFormat(" Num.comm - {0,3}", newPost.NumCommentsScrapped);
if (newPost.Id > 0)
{
newPost.Votes = VotesAnalyzer.ScrapeListVotesForNode(newPost.Id, newPost.Author, "node", inRepo, Browser);
}
output.AppendFormat(" Votes - {0}", newPost.Votes.Count);
log.Info(output.ToString());
newPost.Comments = CommentsAnalyzer.ScrapePostComments(mainContent, inPostUrl, inRepo, inFetchCommentsVotes, Browser);
//Console.WriteLine(" Comments - {0}", newPost.Comments.Count);
return(newPost);
}
public static WebPageModel ToModel(this WebPage entity)
{
return(entity.MapTo <WebPage, WebPageModel>());
}
public static WebPage ToEntity(this WebPageModel model, WebPage destination)
{
return(model.MapTo(destination));
}
public EnhancedHtmlIns(WebPage page, string selector) : base(page, selector) { }
static HtmlNode GetHtml(string url)
{
WebPage webpage = _browser.NavigateToPage(new Uri(url));
return(webpage.Html);
}
private static bool IsAccessibleToUser(WebPage page)
{
//if (node.Equals(RootNode)) return true;
//var page = GetPageFromKey(GetIDFromKey(node.Key));
if (page.AllowAnonymous)
return true;
if (Request.IsAuthenticated)
{
if (roleTable == null)
PreloadRoleTable();
if (roleTable.ContainsKey(page.ID))
{
var roles = roleTable[page.ID];
foreach (var role in roles)
{
if (AppModel.Get().User.IsInRole(role))
return true;
}
}
//page.Roles.Contains(AppModel.Get().User.ro
}
return false;
}
private async Task <HtmlNode> GetHtmlAsync(string url, HttpVerb verb, Field[] data, string contentType)
{
WebPage webpage = await _browser.NavigateToPageAsync(new Uri(url), verb, Serialize(data), contentType);
return(webpage.Html);
}
/// <summary>
/// Construct a form element from the specified web page.
/// </summary>
/// <param name="source">The page that holds this form element.</param>
protected FormElement(WebPage source)
: base(source)
{
}
public void GivenWebPageIsDisplayedOnTheBrowser(string pageName)
{
_inputWebPage = WebPageFactory.CreatePageNamed(pageName);
_inputWebPage.Open();
_inputWebPage.WaitForPageReady();
}
public void InsertWebPage(WebPage page)
{
MongoCollection <WebPage> collection = mongo_database.GetCollection <WebPage>("webpages");
SafeModeResult smr = collection.Insert <WebPage>(page);
}
public ActionResult ManageDocument(Document doc, DocumentFile file, ImageFile thumb, string category, string name, string webPageId, string tmpl)
{
string dbPath = "", dbThumbPath = "";
DBDataContext db = Utils.DB.GetContext();
//check for category <> 0
int categoryId = 0;
if (!string.IsNullOrEmpty(name))
{
//add category to database
Category c = new Category()
{
Name = name
};
try
{
db.Categories.InsertOnSubmit(c);
db.SubmitChanges();
categoryId = c.ID;
}
catch (Exception ex)
{
ModelState.AddModelError("", "An unknown error occurred while adding category to the database.");
ErrorHandler.Report.Exception(ex, "Body/ManageDocument[HTTPPOST]");
}
}
//no new category, try parsing existing one from dropdown list
if (categoryId == 0)
{
Int32.TryParse(category, out categoryId);
}
//validate category
if (categoryId > 0)
{
WebPage page = db.WebPages.SingleOrDefault(x => x.ID == Convert.ToInt32(webPageId));
if (page != null)
{
if (file != null)
{
file.ValidateForUpload(true);
if (ModelState.IsValid)
{
string documentName = "doc-lib_" + DateTime.Now.Ticks.ToString() + "_" + new Random().Next(1, 100).ToString() + System.IO.Path.GetExtension(file.Name);
if (file.Save(documentName))
{
dbPath = "/" + file.SavePath;
file.Cleanup();
}
}
}
else
{
ModelState.AddModelError("", "Document file is not recognized");
}
if (thumb != null)
{
thumb.ValidateForUpload(true);
if (ModelState.IsValid)
{
string thumbName = "doc-lib_thumb_" + DateTime.Now.Ticks.ToString() + "_" + new Random().Next(1, 100).ToString();
if (thumb.Save(thumbName, new System.Drawing.Size(Config.DocLib.Thumbnail.Width, Config.DocLib.Thumbnail.Height), false))
{
dbThumbPath = "/" + thumb.SavePath;
thumb.Cleanup();
}
}
}
else
{
ModelState.AddModelError("", "Image file is not recognized");
}
if (!string.IsNullOrEmpty(dbPath) && !string.IsNullOrEmpty(dbThumbPath))
{
Category c = db.Categories.SingleOrDefault(x => x.ID == categoryId);
Document d = new Document()
{
Title = doc.Title,
Source = dbPath,
Thumb = dbThumbPath
};
d.Category = c;
page.Documents.Add(d);
try
{
db.SubmitChanges();
return(RedirectToAction("DocLib", "Body", new { controller = "Body", action = "DocLib", webpage = webPageId.ToString(), tmpl = tmpl }));
}
catch (Exception ex)
{
ModelState.AddModelError("", "An unknown error occurred. Please try again in a few minutes.");
ErrorHandler.Report.Exception(ex, "Body/ManageDocument[HTTPPOST]");
}
}
else
{
ModelState.AddModelError("", "There has been an issue with uploading your Image file. Please try again in few minutes.");
}
}
else
{
ModelState.AddModelError("", "Page cannot be found. Please try again in few minutes.");
}
}
else
{
ModelState.AddModelError("category", "You must either add new or select valid category for this document.");
}
if (System.IO.File.Exists(HttpContext.Server.MapPath(dbPath)))
{
System.IO.File.Delete(HttpContext.Server.MapPath(dbPath));
}
if (System.IO.File.Exists(HttpContext.Server.MapPath(dbThumbPath)))
{
System.IO.File.Delete(HttpContext.Server.MapPath(dbThumbPath));
}
ViewData["WebPageID"] = webPageId;
ViewData["TemplateID"] = tmpl;
return(View("ManageDocument", new Document()));
}
private void buttonStart_Click(object sender, EventArgs e)
{
var baslangicZamani = DateTime.Now;
saveFileDialog1.Filter = "Excel dosyası|*.xlsx";
saveFileDialog1.DefaultExt = "xlsx";
linkLabel1.Visible = false;
if (saveFileDialog1.ShowDialog() != DialogResult.OK)
{
return;
}
buttonStop.Enabled = true;
buttonStart.Enabled = false;
_adresListesi = new List <Adres>();
_stopDownload = false;
var t = new Task(delegate
{
var caddeSayisi = _caddeler.Count;
Invoke(new MethodInvoker(delegate
{
progressBarCadde.Value = 0;
progressBarCadde.Maximum = caddeSayisi;
}));
ScrapingBrowser browser = new ScrapingBrowser();
browser.Encoding = Encoding.UTF8;
browser.Timeout = TimeSpan.FromSeconds(30);
WebPage homePage = null;
try
{
if (_caddeler != null)
{
foreach (var cadde in _caddeler)
{
if (_stopDownload)
{
break;
}
Invoke(new MethodInvoker(delegate
{
progressBarCadde.Value += 1;
}));
var hucreler = cadde.SelectNodes("td");
var caddeTuru = hucreler[0].InnerText.Replace(" ", "");
var caddeAdi = hucreler[1].InnerText.Replace(" ", "");
var caddekodu = cadde.GetAttributeValue("id", "");
if (caddekodu.Length <= 1)
{
Debug.WriteLine("HHHHHH");
}
else
{
caddekodu = caddekodu.Substring(1);
}
var postData = $"fZwxqvPOpAViHEOXXVuXBaTd+2018072821lryxuVH5Y45L6Yb8Wo05CB46f1vrJzTCd1vDE8EiSLYLbaFSn0O0MhabkTx4t3A+Q==&t=dk&u={caddekodu}&term=";
_binalar = null;
try
{
homePage = browser.NavigateToPage(new Uri("http://adreskodu.dask.gov.tr/site-element/control/load.ashx"),
HttpVerb.Post, postData);
_binalar = homePage.Html.SelectNodes("//tbody/tr");
}
catch (Exception)
{
}
if (_binalar != null)
{
var binaSayisi = _binalar.Count;
Invoke(new MethodInvoker(delegate
{
progressBarBina.Value = 0;
progressBarBina.Maximum = binaSayisi;
}));
foreach (var bina in _binalar)
{
if (_stopDownload)
{
break;
}
var hucreler2 = bina.SelectNodes("td");
var binaNo = hucreler2[0].InnerText.Replace(" ", "");
var binaKodu = hucreler2[1].InnerText.Replace(" ", "");
var siteAdi = hucreler2[2].InnerText.Replace(" ", "");
var apartmanAdi = hucreler2[3].InnerText.Replace(" ", "");
Invoke(new MethodInvoker(delegate
{
progressBarBina.Value += 1;
}));
var binakodu = bina.GetAttributeValue("id", "");
if (binaKodu.Length <= 1)
{
Debug.WriteLine("HHHHHH");
}
else
{
binakodu = binaKodu.Substring(1);
}
postData = $"fZwxqvPOpAViHEOXXVuXBaTd+2018072821lryxuVH5Y45L6Yb8Wo05CB46f1vrJzTCd1vDE8EiSLYLbaFSn0O0MhabkTx4t3A+Q==&t=ick&u={binakodu}&term=";
_daireler = null;
try
{
homePage = browser.NavigateToPage(new Uri("http://adreskodu.dask.gov.tr/site-element/control/load.ashx"),
HttpVerb.Post, postData);
_daireler = homePage.Html.SelectNodes("//tbody/tr");
}
catch (Exception)
{
}
if (_daireler != null)
{
foreach (var daire in _daireler)
{
if (_stopDownload)
{
break;
}
var hucreler3 = daire.SelectNodes("td");
var daireTuru = hucreler3.Count >= 1 ? hucreler3[0].InnerText.Replace(" ", "") : "";
var icKapiNo = hucreler3.Count >= 2 ? hucreler3[1].InnerText.Replace(" ", "") : "";
var adresKodu = daire.GetAttributeValue("id", "");
if (adresKodu.Length >= 2)
{
adresKodu = adresKodu.Substring(1);
}
//var daireIsmi = hucreler3[0].InnerText + "-" + hucreler3[1].InnerText;
//daireIsmi = daireIsmi.Replace(" ", " ");
//Debug.WriteLine("Adres Kodu: " + daire.GetAttributeValue("id", "").Substring(1));
var adres = new Adres();
adres.ilKodu = _ilKodu;
adres.il = _il;
adres.ilceKodu = _ilceKodu;
adres.ilce = _ilce;
adres.koyKodu = _koyKodu;
adres.koy = _koy;
adres.mahalleKodu = _mahalleKodu;
adres.mahalle = _mahalle;
adres.caddeTuru = caddeTuru;
adres.caddeAdi = caddeAdi;
adres.binaNo = binaNo;
adres.binaKodu = binaKodu;
adres.siteAdi = siteAdi;
adres.apartmanAdi = apartmanAdi;
adres.daireTuru = daireTuru;
adres.icKapiNo = icKapiNo;
adres.adresKodu = adresKodu;
_adresListesi.Add(adres);
}
}
}
}
var gecenZaman = DateTime.Now.Subtract(baslangicZamani);
var birCaddeIcinGecenZaman = (int)(gecenZaman.TotalSeconds / progressBarCadde.Value);
var kalanCadde = progressBarCadde.Maximum - progressBarCadde.Value;
var kalanZaman = kalanCadde * birCaddeIcinGecenZaman;
var kalanZamanTS = TimeSpan.FromSeconds(kalanZaman);
Invoke(new MethodInvoker(delegate
{
labelKalanSure.Text = "Geçen Süre: " + gecenZaman.ToString(@"hh\:mm\:ss") + " Kalan Süre: " + kalanZamanTS.ToString(@"hh\:mm\:ss");
}));
}
}
}
catch (Exception ex)
{
MessageBox.Show("Bilgileri sorgularken hata oluştu:\r\n" + ex.Message, "Hata!", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
try
{
using (var p = new ExcelPackage())
{
//A workbook must have at least on cell, so lets add one...
var ws = p.Workbook.Worksheets.Add("MySheet");
ws.Cells.LoadFromCollection <Adres>(_adresListesi, true);
ws.Cells[ws.Dimension.Address].AutoFitColumns();
//Save the new workbook. We haven't specified the filename so use the Save as method.
p.SaveAs(new FileInfo(saveFileDialog1.FileName));
Invoke(new MethodInvoker(delegate
{
linkLabel1.Visible = true;
linkLabel1.Text = saveFileDialog1.FileName;
}));
}
}
catch (Exception ex)
{
MessageBox.Show("Dosya kaydedilirken hata oluştu:\r\n" + ex.Message, "Hata!", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
Invoke(new MethodInvoker(delegate
{
buttonStart.Enabled = true;
buttonStop.Enabled = false;
}));
});
t.Start();
}
public ActionResult PDFPrev(string webPageId, DocumentFile doc)
{
string code = "";
if (doc != null)
{
doc.ValidateForUpload(true);
if (ModelState.IsValid)
{
if (doc.Save(doc.Name.Replace(" ", "").ToLower()))
{
code = doc.SavedName;
doc.Cleanup();
}
}
}
else
{
ModelState.AddModelError("", "Document File format is not recognized");
}
if (!string.IsNullOrEmpty(code))
{
DBDataContext db = Utils.DB.GetContext();
WebPage pg = db.WebPages.SingleOrDefault(x => x.ID == Convert.ToInt32(webPageId));
if (pg != null)
{
Body b = pg.Bodies.FirstOrDefault();
if (b != null)
{
//delete existing PDF file
if (System.IO.File.Exists(HttpContext.Server.MapPath(Url.Data(b.HTML))))
{
System.IO.File.Delete(HttpContext.Server.MapPath(Url.Data(b.HTML)));
}
b.HTML = HttpUtility.HtmlEncode(code);
}
else
{
pg.Bodies.Add(new Body()
{
HTML = HttpUtility.HtmlEncode(code)
});
}
try
{
db.SubmitChanges();
return(RedirectToAction("Index", "WebPage", new { controller = "WebPage", action = "Index", webpage = webPageId }));
}
catch (Exception ex)
{
ModelState.AddModelError("", ex.Message);
ErrorHandler.Report.Exception(ex, "Body/PDFPrev [POST]");
}
}
else
{
ModelState.AddModelError("", "Web Page does not exist");
}
}
else
{
ModelState.AddModelError("", "There has been an issue with uploading your Document file. Please try again in few minutes.");
}
return(View());
}
/// <summary>
/// Construct this Input element.
/// </summary>
/// <param name="source">The source for this input ent.</param>
public Input(WebPage source)
: base(source)
{
}
public override bool IsLoadedCorrectly(WebPage page)
{
return(page.Html.CssSelect("#accordion").Any());
}
/// <summary>
/// Retrieves a string displaying the amount of characters, lines, and search term occurances in a string
/// </summary>
/// <param name="texts"></param>
/// <returns></returns>
public WebPage constructWebsite(Queue <string> texts, Queue <string> links, Queue <string> images, string url, string host)
{
filterDuplicates(texts);
filterDuplicates(links);
filterDuplicates(images);
int chars = 0;
int phraseMatches = 0;
int words = 0;
int tokenMatches = 0;
string sample = String.Empty;
for (int j = 0; j < texts.Count; j++)
{
string line = texts.Dequeue();
chars += line.Length;
words += line.getChunks();
// Use the current line and fine the phraseMatchs and term matches
for (int i = 0; i < searchTokens.Count; i++)
{
// check when it is the search phrase
if (line.Contains(searchTokens[i]) && searchTokens[i].Equals(searchPhrase))
{
phraseMatches += line.Occurrences(searchTokens[i]);
string phrase = line.Substring(line.IndexOf(searchPhrase));
if (phrase.Length > sample.Length)
{
sample = phrase;
}
}
if (line.Contains(searchTokens[i]))
{
tokenMatches += line.Occurrences(searchPhrase);
}
}
texts.Enqueue(line);
}
for (int i = 0; i < searchTokens.Count; i++)
{
if (url.Contains(searchTokens[i]))
{
tokenMatches++;
}
if (host.Contains(searchTokens[i]))
{
tokenMatches++;
}
}
if (url.Contains(searchPhrase))
{
phraseMatches++;
}
WebPage page = new WebPage();
page.setUrl(url);
page.setCharacterCount(chars);
page.setLineCount(texts.Count);
page.setSearchPhraseCount(phraseMatches);
page.setSearchTokensCount(tokenMatches);
page.setInboundUrls(getHashesOfInboundUrls(links));
page.setImageCount(images.Count);
page.setWordCount(words);
// remove new lines if any found
page.setSampleText(sample.Replace("\n", ""));
page.setHost(host);
return(page);
}
public void OpenWebPage(WebPage webPage)
{
}
} // { get => nodes; private set => SetValueAndNotify(ref nodes, value, nameof(HtmlNodes)); }
public BlackWhiteListWindow(WebPage webPage)
{
WebPage = webPage;
InitializeComponent();
//VirtualizingPanel.SetIsVirtualizing(tree, false);
}
/// <summary>
/// Construct a span range from the specified web page.
/// </summary>
/// <param name="source">The source web page.</param>
public Span(WebPage source)
: base(source)
{
}
/// <summary>
/// Construct this Input element.
/// </summary>
/// <param name="source">The source for this input ent.</param>
public Input(WebPage source)
: base(source)
{
}
/// <summary>
/// 生成一个引用JS文件的HTML代码，其中URL包含了文件的最后更新时间。
/// </summary>
/// <param name="page">当前页面</param>
/// <param name="jsPath">要引用的JS文件路径</param>
/// <returns></returns>
public static IHtmlString RenderJs(this WebPage page, string jsPath)
{
string html = UiHelper.RefJsFileHtml(jsPath);
return(new HtmlString(html));
}
public Recipe Parse()
{
Recipe output = new Recipe();
output.Ingredients = new List <string>();
output.Steps = new List <string>();
output.ImageLocations = new List <string>();
string titleNode = "";
string descriptionNode = "";
string imgNode = "";
string stepsNode = "";
string ingredientNode = "";
string prepTimeNode = "";
string cookTimeNode = "";
string website = "";
HtmlWeb web = new HtmlWeb();
HtmlDocument doc = web.Load(URL);
HtmlNode node;
if (URL.Contains("allrecipes.com"))
{
website = "allrecipes.com";
output.Source = "All Recipes";
titleNode = "//h1[@itemprop='name']";
descriptionNode = "//div[@itemprop='description']";
imgNode = "//img[@class='rec-photo']";
stepsNode = "//li[@class='step']";
ingredientNode = "//span[@itemprop='recipeIngredient']";
prepTimeNode = "//time[@itemprop='prepTime']";
cookTimeNode = "//time[@itemprop='cookTime']";
node = doc.DocumentNode.SelectSingleNode(prepTimeNode);
HtmlNodeCollection prepTime = node.FirstChild.ChildNodes;
int min = 0;
int hour = 0;
if (prepTime.Count() > 2)
{
hour = Convert.ToInt32(prepTime[0].InnerText);
min = Convert.ToInt32(prepTime[1].InnerText);
}
else
{
min = Convert.ToInt32(prepTime[0].InnerText);
}
output.PrepMinutes = (hour * 60) + min;
node = doc.DocumentNode.SelectSingleNode(cookTimeNode);
HtmlNodeCollection cookTime = node.FirstChild.ChildNodes;
min = 0;
hour = 0;
if (cookTime.Count() > 2)
{
hour = Convert.ToInt32(cookTime[0].InnerText);
min = Convert.ToInt32(cookTime[2].InnerText);
}
else
{
min = Convert.ToInt32(cookTime[0].InnerText);
}
output.CookMinutes = (hour * 60) + min;
node = doc.DocumentNode.SelectSingleNode(titleNode);
if (node != null)
{
output.Name = node.InnerText;
}
node = doc.DocumentNode.SelectSingleNode(descriptionNode);
if (node != null)
{
output.Description = node.InnerText;
}
while (output.Description.Contains("""))
{
output.Description = output.Description.Remove(output.Description.IndexOf("""), 5);
}
output.Description.Trim();
if (output.Description[0] == ' ')
{
output.Description.Remove(0, 1);
}
node = doc.DocumentNode.SelectSingleNode(imgNode);
List <string> images = new List <string>();
if (node != null)
{
images.Add(node.GetAttributeValue("src", ""));
}
output.ImageLocations = images;
output.Source = URL;
output.DateAdded = DateTime.Now;
HtmlNodeCollection stepNodes = doc.DocumentNode.SelectNodes(stepsNode);
List <string> steps = new List <string>();
foreach (HtmlNode step in stepNodes)
{
steps.Add(step.InnerText);
if (String.IsNullOrWhiteSpace(steps[steps.Count - 1]))
{
steps.RemoveAt(steps.Count - 1);
}
}
output.Steps = steps;
HtmlNodeCollection ingredientNodes = doc.DocumentNode.SelectNodes(ingredientNode);
List <string> ingredients = new List <string>();
foreach (HtmlNode ingredient in ingredientNodes)
{
ingredients.Add(ingredient.InnerText);
}
output.Ingredients = ingredients;
}
else if (URL.Contains("foodnetwork.com"))
{
website = "foodnetwork.com";
//titleNode = "//span[@class='o - AssetTitle__a - HeadlineText']";
//descriptionNode = "//span[@class='originalText']";
//imgNode = "//img[@class='m-MediaBlock__a-Image a-Image']";
//stepsNode = "//div[@class='0-Method__m-Body']";
//ingredientNode = "//div[@class='0-Ingredients__m-Body']";
//prepTimeNode = "na";
//cookTimeNode = "na";
ScrapingBrowser browser = new ScrapingBrowser();
//set UseDefaultCookiesParser as false if a website returns invalid cookies format
//browser.UseDefaultCookiesParser = false;
WebPage homePage = browser.NavigateToPage(new Uri(URL));
HtmlNode[] nodes = homePage.Html.CssSelect("span.o-AssetTitle__a-HeadlineText").ToArray();
if (nodes.Length > 0)
{
output.Name = nodes[0].InnerText;
}
else
{
output.Name = "Could not find name";
}
nodes = homePage.Html.CssSelect("div.o-AssetDescription__a-Description").ToArray();
if (nodes.Length > 0)
{
output.Description = nodes[0].ChildNodes[0].InnerText;
}
else
{
output.Description = "Could not find Description";
}
nodes = homePage.Html.CssSelect("img.m-MediaBlock__a-Image").ToArray();
if (nodes.Length > 0)
{
List <string> images = new List <string>();
images.Add(nodes[0].GetAttributeValue("src"));
output.ImageLocations = images;
}
nodes = homePage.Html.CssSelect("p.o-Ingredients__a-Ingredient").ToArray();
List <string> ingredients = new List <string>();
if (nodes.Length > 0)
{
foreach (HtmlNode ing in nodes)
{
ingredients.Add(ing.InnerText);
}
}
else
{
ingredients.Add("Error finding ingredients");
}
output.Ingredients = ingredients;
nodes = homePage.Html.CssSelect("li.o-Method__m-Step").ToArray();
List <string> steps = new List <string>();
if (nodes.Length > 0)
{
foreach (HtmlNode step in nodes)
{
steps.Add(step.InnerText);
}
}
else
{
steps.Add("Error finding steps");
}
output.Steps = steps;
nodes = homePage.Html.CssSelect("ul.o-RecipeInfo__m-Time").ToArray();
string time = nodes[0].ChildNodes[3].ChildNodes[3].InnerText.ToLower();
int minutes = 0;
if (time.Contains("hr"))
{
string hours = time.Substring(0, time.IndexOf("hr"));
hours.Trim();
minutes += Convert.ToInt32(hours);
minutes *= 60;
time = time.Substring(time.IndexOf("hr") + 3);
time = time.Trim();
}
if (time.Contains("min"))
{
time = time.Remove(time.IndexOf("min"), 3);
time = time.Trim();
}
output.PrepMinutes = Convert.ToInt32(time) + minutes;
minutes = 0;
time = nodes[0].ChildNodes[1].ChildNodes[3].InnerText.ToLower();
if (time.Contains("hr"))
{
string hours = time.Substring(0, time.IndexOf("hr"));
time = time.Remove(0, time.IndexOf("hr") + 2);
time = time.Trim();
hours = hours.Trim();
minutes += Convert.ToInt32(hours);
minutes *= 60;
}
if (time.Contains("min"))
{
time = time.Remove(time.IndexOf("min"), 3);
time = time.Trim();
}
output.CookMinutes = (Convert.ToInt32(time) + minutes) - output.PrepMinutes;
}
else if (URL.Contains("myrecipes.com/recipe"))
{
website = "myrecipes.com";
ScrapingBrowser browser = new ScrapingBrowser();
WebPage homePage = browser.NavigateToPage(new Uri(URL));
node = homePage.Html.CssSelect("h1.headline").Single();
output.Name = node.InnerText;
node = homePage.Html.CssSelect("div.recipe-summary").Single();
node = node.ChildNodes[1];
output.Description = node.InnerText;
node = homePage.Html.CssSelect("div.ingredients").Single();
node = node.ChildNodes[1];
for (int i = 1; i < node.ChildNodes.Count; i += 2)
{
output.Ingredients.Add(node.ChildNodes[i].InnerText);
}
HtmlNode[] nodes = homePage.Html.CssSelect("div.step").ToArray();
foreach (HtmlNode stepDiv in nodes)
{
foreach (HtmlNode step in stepDiv.ChildNodes)
{
if (step.Name == "p")
{
output.Steps.Add(step.InnerText);
}
}
}
nodes = homePage.Html.CssSelect("div.recipe-meta-item-body").ToArray();
string time = nodes[0].InnerText.ToLower();
int minutes = 0;
if (time.ToLower().Contains("hours"))
{
string hours = time.Substring(0, time.IndexOf("hours"));
hours.Trim();
minutes += Convert.ToInt32(hours);
minutes *= 60;
time = time.Substring(time.IndexOf("hours") + 6);
time = time.Trim();
}
if (time.ToLower().Contains("hour"))
{
string hours = time.Substring(0, time.IndexOf("hour"));
hours.Trim();
minutes += Convert.ToInt32(hours);
minutes *= 60;
time = time.Substring(time.IndexOf("hour") + 5);
time = time.Trim();
}
if (time.ToLower().Contains("min"))
{
time = time.Remove(time.IndexOf("min"), 4);
time = time.Trim();
}
output.PrepMinutes = Convert.ToInt32(time) + minutes;
minutes = 0;
time = nodes[1].InnerText.ToLower();
if (time.ToLower().Contains("hours"))
{
string hours = time.Substring(0, time.IndexOf("hours"));
hours.Trim();
minutes += Convert.ToInt32(hours);
minutes *= 60;
time = time.Substring(time.IndexOf("hours") + 6);
time = time.Trim();
}
if (time.ToLower().Contains("hour"))
{
string hours = time.Substring(0, time.IndexOf("hour"));
time = time.Remove(0, time.IndexOf("hour") + 5);
time = time.Trim();
hours = hours.Trim();
minutes += Convert.ToInt32(hours);
minutes *= 60;
}
if (time.ToLower().Contains("min"))
{
time = time.Remove(time.IndexOf("min"), 4);
time = time.Trim();
}
output.CookMinutes = (Convert.ToInt32(time) + minutes) - output.PrepMinutes;
}
else if (URL.Contains("tasteofhome.com"))
{
output.Source = "Taste of Home";
throw new NotImplementedException();
}
else if (URL.Contains("delish.com"))
{
output.Source = "Delish";
throw new NotImplementedException();
}
else if (URL.Contains("skinnytaste.com"))
{
output.Source = "Skinny Taste";
throw new NotImplementedException();
}
else if (URL.Contains("thekitchn.com"))
{
output.Source = "The Kitchn";
throw new NotImplementedException();
}
for (int i = 0; i < output.Steps.Count; i++)
{
output.Steps[i] = output.Steps[i].Trim();
}
for (int i = 0; i < output.Ingredients.Count; i++)
{
output.Ingredients[i] = output.Ingredients[i].Trim();
}
output.DateAdded = DateTime.Now;
output.Source = URL;
return(output);
}
private static void RenderNode(HtmlTextWriter writer, WebPage p, IEnumerable<WebPage> pages, bool withAttrs = true, bool showImg = true, WebPage curPage = null, int dataType = 1)
{
//var urlHelper = new UrlHelper(Context.Request);
var path = string.Format("~/{0}/{1}/{2}.html", p.Web.Name, p.Locale, p.Slug);
writer.WriteBeginTag("li");
if (!object.ReferenceEquals(curPage, null) && !object.ReferenceEquals(p.Path, null) && !object.ReferenceEquals(curPage.Path, null))
{
var parentPath = p.Path + "/";
if (curPage.Path.StartsWith(parentPath, StringComparison.OrdinalIgnoreCase) || p.ID == curPage.ID)
writer.WriteAttribute("class", "d-state-active");
}
writer.Write(HtmlTextWriter.TagRightChar);
writer.WriteBeginTag("a");
writer.WriteAttribute("data-id", p.ID.ToString());
if (withAttrs)
{
writer.WriteAttribute("data-parentId", p.ParentID.ToString());
writer.WriteAttribute("data-in-menu", p.ShowInMenu.ToString().ToLower());
writer.WriteAttribute("data-in-sitemap", p.ShowInSitemap.ToString().ToLower());
writer.WriteAttribute("data-static", p.IsStatic.ToString().ToLower());
writer.WriteAttribute("data-shared", p.IsShared.ToString().ToLower());
writer.WriteAttribute("data-title", p.Title);
writer.WriteAttribute("data-desc", p.Description);
//writer.WriteAttribute("data-slug", p.Slug.ToLower());
writer.WriteAttribute("data-path", Href(path));
writer.WriteAttribute("data-anonymous", p.AllowAnonymous.ToString().ToLower());
}
else
{
if (p.NoFollow)
writer.WriteAttribute("rel", "nofollow");
else
{
if (!string.IsNullOrEmpty(Request.Browser.Browser) && Request.Browser.Browser.Equals("chrome",StringComparison.OrdinalIgnoreCase))
writer.WriteAttribute("rel", "preload");
else
writer.WriteAttribute("rel", "prefetch");
}
if (!string.IsNullOrEmpty(p.LinkTo))
writer.WriteAttribute("href", p.LinkTo);
else
writer.WriteAttribute("href", Href(path));
}
writer.Write(HtmlTextWriter.TagRightChar);
if (showImg)
{
if (!string.IsNullOrEmpty(p.IconUrl))
{
writer.WriteBeginTag("img");
writer.WriteAttribute("src", Href(p.IconUrl));
writer.WriteAttribute("alt", p.Title);
writer.Write(HtmlTextWriter.SelfClosingTagEnd);
}
}
writer.WriteEncodedText(GE.GetContent(p.Title));
writer.WriteEndTag("a");
if (dataType == 1)
{
var children = pages.Where(page => page.ParentID == p.ID).OrderBy(page => page.Pos).ToList();
if (children.Count > 0)
{
writer.WriteFullBeginTag("ul");
foreach (var child in children)
{
RenderNode(writer, child, pages, withAttrs, showImg);
}
writer.WriteEndTag("ul");
}
}
writer.WriteEndTag("li");
}
/// <summary>
/// 生成一个引用CSS文件的HTML代码，其中URL包含了文件的最后更新时间。
/// </summary>
/// <param name="page">当前页面</param>
/// <param name="cssPath">要引用的CSS文件路径</param>
/// <returns></returns>
public static IHtmlString RenderCss(this WebPage page, string cssPath)
{
string html = UiHelper.RefCssFileHtml(cssPath);
return(new HtmlString(html));
}
/// <summary>
/// Construct a link from the specified web page.
/// </summary>
/// <param name="source">The web page this link is from.</param>
public Link(WebPage source)
: base(source)
{
}
private static HtmlNode GetHtml(string url)
{
WebPage webPage = _scrapingBrowser.NavigateToPage(new Uri(url));
return(webPage.Html);
}
/// <summary>
/// Construct a form on the specified web page.
/// </summary>
/// <param name="source">The web page that contains this form.</param>
public Form(WebPage source)
: base(source)
{
}
public override TModel TranslateEntityToVm(WebPage entity)
{
return(CreateBaseEntityVmDefinitions(entity).GetFinal());
}
/// <summary>
/// Construct a range to hold the DIV tag.
/// </summary>
/// <param name="source">The web page this range was found on.</param>
public Div(WebPage source)
: base(source)
{
}
protected ITranslationDefinitions <WebPage, TModel> CreateBaseEntityVmDefinitions(WebPage entity)
{
return(CreateEntityViewModelDefinition <TModel>(entity)
.AddSimple(i => i.Id, o => o.Id)
.AddNavigation(i => i.Name, o => o.Value)
.AddNavigation(i => i.Url, o => o.Url)
.AddNavigation(i => languageCache.GetByValue(i.LocalizationId), o => o.Language));
}
/// <summary>
/// Construct a link from the specified web page.
/// </summary>
/// <param name="source">The web page this link is from.</param>
public Link(WebPage source)
: base(source)
{
}
private void SaveCrew()
{
String Sql = "";
String JobDate = "";
try
{
Sql = "Select JobDate From mcJobs Where JobRno = " + JobRno;
JobDate = Fmt.Dt(db.SqlDtTm(Sql));
}
catch (Exception Ex)
{
Err Err = new Err(Ex, Sql);
Response.Write(Err.Html());
}
for (int iCrew = 1; iCrew <= cCrew; iCrew++)
{
TableRow tr = tblCrew.Rows[iCrew];
CheckBox chkRemove = (CheckBox)tr.FindControl("chkRemove" + iCrew);
if (chkRemove != null && !chkRemove.Checked)
{
Int32 CrewSeq = Str.Num(WebPage.FindTextBox(ref tr, "txtCrewSeq" + iCrew));
String CrewMember = WebPage.FindTextBox(ref tr, "txtCrewMember" + iCrew);
String OrigCrewMember = WebPage.FindTextBox(ref tr, "txtOrigCrewMember" + iCrew);
String Assignment = WebPage.FindTextBox(ref tr, "txtAssignment" + iCrew);
String OrigAssignment = WebPage.FindTextBox(ref tr, "txtOrigAssignment" + iCrew);
String ReportTime = WebPage.FindTextBox(ref tr, "txtReportTime" + iCrew);
String OrigReportTime = WebPage.FindTextBox(ref tr, "txtOrigReportTime" + iCrew);
String Note = WebPage.FindTextBox(ref tr, "txtNote" + iCrew);
String OrigNote = WebPage.FindTextBox(ref tr, "txtOrigNote" + iCrew);
if (CrewMember != OrigCrewMember ||
Assignment != OrigAssignment ||
ReportTime != OrigReportTime ||
Note != OrigNote)
{
DateTime Tm = DateTime.Now;
try
{
if (CrewSeq == 0)
{
CrewSeq = db.NextSeq("mcJobCrew", "JobRno", JobRno, "CrewSeq");
Sql =
"Insert Into mcJobCrew (JobRno, CrewSeq, CreatedDtTm, CreatedUser) Values (" +
JobRno + ", " +
CrewSeq + ", " +
DB.PutDtTm(Tm) + ", " +
DB.PutStr(g.User) + ")";
db.Exec(Sql);
}
Sql =
"Update mcJobCrew Set " +
"CrewMember = " + DB.PutStr(CrewMember, 50) + ", " +
"CrewAssignment = " + DB.PutStr(Assignment, 50) + ", " +
"ReportTime = " + DB.PutDtTm(JobDate, ReportTime) + ", " +
"Note = " + DB.PutStr(Note, 128) + ", " +
"UpdatedDtTm = " + DB.PutDtTm(Tm) + ", " +
"UpdatedUser = "******" " +
"Where JobRno = " + JobRno + " " +
"And CrewSeq = " + CrewSeq;
db.Exec(Sql);
}
catch (Exception Ex)
{
Err Err = new Err(Ex, Sql);
Response.Write(Err.Html());
}
}
}
}
}
