public ActionResult UpdateWebSite(int id)
{
WebSiteOperation uo = new WebSiteOperation();
WebSiteModel u = uo.Get(id);
return(View(u));
}
public void ParseAllLinksFromWebSite(WebSiteModel baseWebSite)
{
try
{
var baseUriObj = new Uri(baseWebSite.URI);
html = new WebClient().DownloadString(baseUriObj);
}
catch (Exception e)
{
_exceptionNotificationServiceManager.ExceptionNotify(e.ToString());
}
foreach (Match match in regExprToParseAllLinks.Matches(html))
{
var rawLink = match.Groups[1].ToString();
var containedUriObj = new Uri(rawLink, UriKind.RelativeOrAbsolute);
if (containedUriObj.IsAbsoluteUri)
{
baseWebSite.AddToContaiedLinks(rawLink);
}
else
{
baseWebSite.AddToContaiedLinks(baseWebSite.RootUrl + rawLink);
}
}
}
public void ModelToEntity()
{
// arrange
var model = new WebSiteModel
{
Category = new CategoryModel {
Id = 2, Name = "Entertainment"
},
Name = "Name",
Url = "https://www.google.com",
ScreenshotUrl = "whatever",
Login = new LoginModel
{
Email = "*****@*****.**",
Password = "******"
}
};
// act
var mapper = new WebSiteMapper(new CategoryMapper(), new CryptoService());
WebSite entity = mapper.ModelToEntity(model);
// assert
Assert.IsNotNull(entity);
Assert.AreEqual(model.Name, entity.Name);
Assert.AreEqual(model.Category.Id, entity.Category.Id);
Assert.AreEqual(model.Category.Name, entity.Category.Name);
Assert.AreEqual(model.Url, entity.Url);
Assert.AreEqual(model.Login.Email, entity.Email);
Assert.AreNotEqual(model.Login.Password, entity.Password);
Assert.AreEqual("Ef1PI4JaZxN6DJSS5MWInW8wOUJjcTscXy5SH9r9vnc=", entity.Password);
}
public void EntityToModel()
{
// arrange
var entity = new WebSite
{
Id = 1,
HasScreenshot = true,
ScreenshotExt = ".whatever",
Category = new WebSiteCategory(2, "Entertainment"),
Email = "*****@*****.**",
Password = "******",
Name = "Name",
Url = "http://some.url"
};
// act
var mapper = new WebSiteMapper(new CategoryMapper(), new CryptoService());
WebSiteModel model = mapper.EntityToModel(entity);
// assert
Assert.IsNotNull(model);
Assert.AreEqual(entity.Url, model.Url);
Assert.AreEqual(entity.Name, model.Name);
Assert.AreEqual("totally valid password", model.Login.Password);
Assert.AreEqual(entity.Email, model.Login.Email);
Assert.AreEqual(entity.Category.Name, model.Category.Name);
Assert.AreEqual(entity.Category.Id, model.Category.Id);
Assert.AreEqual("/api/website/1/screenshot", model.ScreenshotUrl);
}
public ActionResult Index(WebSiteModel model)
{
try
{
Configs.SetValue("MaxCommentNum", model.MaxCommentNum);
Configs.SetValue("MaxFeedbackNum", model.MaxFeedbackNum);
Configs.SetValue("OpenComment", model.OpenComment == "on" ? "true" : "false");
Configs.SetValue("OpenFeedback", model.OpenFeedback == "on" ? "true" : "false");
Configs.SetValue("SiteName", model.SiteName);
Configs.SetValue("SiteTitle", model.SiteTitle);
Configs.SetValue("SiteDomain", model.SiteDomain);
Configs.SetValue("QQ", model.QQ);
Configs.SetValue("Mail", model.Mail);
Configs.SetValue("Address", model.Address);
Configs.SetValue("Gitee", model.Gitee);
Configs.SetValue("MetaKey", model.MetaKey);
Configs.SetValue("MetaDescribe", model.MetaDescribe);
Configs.SetValue("MaxFileUpload", model.MaxFileUpload);
Configs.SetValue("HomeTitle", model.HomeTitle);
Configs.SetValue("CacheTime", model.CacheTime);
Configs.SetValue("CopyRight", model.CopyRight);
Configs.SetValue("UploadFileType", model.UploadFileType);
}
catch (Exception ex)
{
ViewBag.Msg = "Error:" + ex.Message;
return(View(GetWebSiteInfo()));
}
ViewBag.Msg = "修改成功！";
return(View(GetWebSiteInfo()));
}
public ActionResult UpdateWebSiteSub(WebSiteModel model)
{
WebSiteOperation uo = new WebSiteOperation();
uo.Update(model);
DirFile.CreateDirectory(model.Path);
return(JavaScript("pagesub();"));
}
public ViewViewComponentResult Invoke()
{
var model = new WebSiteModel {
definition = _definition
};
return(View(model));
}
/// <summary>
/// 获取cookie值
/// </summary>
/// <param name="model"></param>
private string GetCookie(WebSiteModel model)
{
var result = http.GetHtml(new HttpItem()
{
URL = model.SiteDomin
});
return result.Cookie;
}
public ViewViewComponentResult Invoke()
{
var model = new WebSiteModel
{
services = _pageService.GetAll(PageType.Services),
tours = _pageService.GetAll(PageType.Tour)
};
return(View(model));
}
private void btnTest_Click(object sender, EventArgs e)
{
if (cbWebSite.SelectedItem == null || txtURL.Text == "" || WebSiteContext.GetModel(((WebSiteModel)cbWebSite.SelectedItem).Id) == null)
{
MessageBox.Show("Lutfen gerekli alanlari oldurun!", "Uyari", MessageBoxButtons.OK);
return;
}
if (DataContext.IsSearchPointExist(txtURL.Text))
{
MessageBox.Show("Ilan hali hazirlda kayitli!", "Uyari", MessageBoxButtons.OK);
return;
}
WebSiteModel site = (WebSiteModel)cbWebSite.SelectedItem;
switch (site.Id)
{
case 1:
Rec = SC.HPScanAd(txtURL.Text);
break;
case 2:
Rec = SC.N11ScanAd(txtURL.Text);
break;
default:
break;
}
if (Rec == null)
{
lblTestStatus.ForeColor = Color.Red;
lblTestStatus.Text = "Operation failed!";
btnAdd.Enabled = false;
}
else
{
if (txtName.Text != "")
{
string tempText = txtName.Text;
FillForm();
txtName.Text = tempText;
Rec.Name = tempText;
}
else
{
FillForm();
}
lblTestStatus.ForeColor = Color.Green;
lblTestStatus.Text = "Operation succecful!";
btnAdd.Enabled = true;
}
}
private WebSiteModel DownloadWebsite(string url)
{
WebClient webClient = new WebClient();
WebSiteModel webSiteModel = new WebSiteModel()
{
Url = url,
Data = webClient.DownloadString(url)
};
return(webSiteModel);
}
public ActionResult SiteMap()
{
var model = new WebSiteModel
{
services = _pageService.GetAll(PageType.Services),
tours = _pageService.GetAll(PageType.Tour)
};
Response.ContentType = "xml";
return(View(model));
}
//2
private async Task <WebSiteModel> DownloadWebsiteAsync(string url)
{
WebClient webClient = new WebClient();
WebSiteModel webSiteModel = new WebSiteModel()
{
Url = url,
Data = await webClient?.DownloadStringTaskAsync(url)
};
return(webSiteModel);
}
public static WebSiteModel CreateModel(List<UrlModel> downloadUrls, RuleModel rule,
Uri sourceAddress)
{
WebSiteModel model = new WebSiteModel();
model.DownloadUrls = downloadUrls;
model.Rule = rule;
model.SourceAddress = sourceAddress;
return model;
}
public void WriteReport(WebSiteModel WebSiteToReport)
{
var csv = new StringBuilder();
csv.AppendLine(WebSiteToReport.ToString());
foreach (var containedLink in WebSiteToReport.ContaiedLinks)
{
csv.AppendLine(containedLink.ToString());
}
csv.AppendLine("NEXT WEBSITE LINKS, NEXT WEBSITE LINKS");
File.AppendAllText(reportPath, csv.ToString());
}
public ActionResult InsertWebSiteSub(WebSiteModel u)
{
WebSiteOperation uo = new WebSiteOperation();
u.CreateTime = DateTime.Now;
uo.Insert(u);
DirFile.CreateDirectory(u.Path);
return(JavaScript("pagesub();"));
}
public async Task <IActionResult> Post([FromBody] WebSiteModel value)
{
if (!ModelState.IsValid)
{
return(ValidationProblem(ModelState));
}
WebSite webSite = _webSiteMapper.ModelToEntity(value);
int id = await _webSiteService.CreateWebSite(webSite);
return(Ok(id));
}
public ShellViewModel()
{
DataAccess da = new DataAccess();
Cities = new BindableCollection <CityModel>(da.GetCityModelList());
Professions = new BindableCollection <ProfessionModel>(da.GetProfessionModelList());
Websites = new BindableCollection <WebModel>(da.GetWebistesModelList());
Numbers = new BindableCollection <NumbersModel>(da.GetNumberModelList());
ExtractedContent = new BindableCollection <WebSiteModel> {
};
SelectedCredentialsFromWeb = new WebSiteModel {
};
}
private HttpResult PostLogin(WebSiteModel model)
{
string postData = string.Format("action=user_login&user_login={0}&user_pass={1}&remember_me=1&redirect_url={2}",
model.LoginName, model.LoginPwd, model.SignRequestUrl.UrlEncode());
HttpItem item = new HttpItem()
{
URL = model.LoginUrl,
Postdata = postData,
Cookie = model.Cookie,
Referer = model.SiteDomin,
Method = "POST",
Accept = "application/json, text/javascript, */*",// 可选项有默认值
ContentType = "application/x-www-form-urlencoded",//返回类型 可选项有默认值
};
var result = http.GetHtml(item);
return result;
}
static int Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Please enter a link to the config file and a valid url to the website");
return(0);
}
var configPath = args[0];
var uri = args[1];
var ninjectInitializer = new NinjectInitializer();
ninjectInitializer.InitNinject(configPath,
out Settings userSettings,
out IExceptionNotificationService exceptionNotificationServiceManager,
out IParserService parserServiceManager,
out IReportService reportServiceManager,
out IWebSiteStatusInspector webSiteStatusInspectorManager);
var mainLink = new WebSiteModel(uri, userSettings.Nesting);
mainLink.StatusCode = webSiteStatusInspectorManager.CheckWebsiteStatus(mainLink.URI);
if (mainLink.StatusCode.ToString().StartsWith("2") || mainLink.StatusCode.ToString().StartsWith("3"))
{
Console.WriteLine($"Program In Progress...");
var eachLinkProcedure = new MainProcedure(parserServiceManager,
webSiteStatusInspectorManager, reportServiceManager);
try
{
eachLinkProcedure.StartMainProcedure(mainLink);
}
catch (Exception e)
{
exceptionNotificationServiceManager.ExceptionNotify(e.ToString());
}
}
else
{
Console.WriteLine($"Main link is not avialable");
return(0);
}
Console.WriteLine("DONE");
return(1);
}
public ActionResult Modify(WebSiteModel entity)
{
if (ModelState.IsValid)
{
if (ConfigHelper.SaveConfig(ConfigFileEnum.SiteConfig, entity))
{
AddAdminLogs(Entity.SysLogMethodType.Update, "修改配置文件");
return(PromptView("/admin/SysConfig/Modify", "OK", "Success", "修改成功", 5));
}
else
{
return(PromptView("/admin/SysConfig/Modify", "500", "Error", "保存失败，可能是没有权限操作数据库", 5));
}
}
else
{
return(View(entity));
}
}
public async Task <IActionResult> Put(int webSiteId, [FromBody] WebSiteModel value)
{
if (!ModelState.IsValid)
{
return(ValidationProblem(ModelState));
}
WebSite webSite = _webSiteMapper.ModelToEntity(value);
webSite.Id = webSiteId;
bool success = await _webSiteService.UpdateWebSite(webSite);
if (!success)
{
return(NotFound());
}
return(Ok());
}
public void StartMainProcedure(WebSiteModel baseWebSite)
{
_parserServiceManager.ParseAllLinksFromWebSite(baseWebSite);
Console.WriteLine($"Working with {baseWebSite.URI}");
foreach (var containedLink in baseWebSite.ContaiedLinks)
{
containedLink.StatusCode = _webSiteStatusInspectorManager.CheckWebsiteStatus(containedLink.URI);
}
_reportServiceManager.WriteReport(baseWebSite);
if (baseWebSite.Nesting >= 2)
{
foreach (var containedLink in baseWebSite.ContaiedLinks)
{
if (containedLink.StatusCode.ToString().StartsWith("2") || containedLink.StatusCode.ToString().StartsWith("3"))
{
StartMainProcedure(containedLink);
}
}
}
}
public void EntityToModel__When_has_no_screenshot__ScreenshotUrl_is_null()
{
// arrange
var entity = new WebSite
{
Id = 1,
HasScreenshot = false,
ScreenshotExt = null,
Category = new WebSiteCategory(2, "Entertainment"),
Email = "*****@*****.**",
Password = "******",
Name = "Name",
Url = "http://some.url"
};
// act
var mapper = new WebSiteMapper(new CategoryMapper(), new CryptoService());
WebSiteModel model = mapper.EntityToModel(entity);
// assert
Assert.IsNull(model.ScreenshotUrl);
}
/// <summary>
/// 初始化调用构造函数后可能不可用的数据
/// </summary>
/// <param name="requestContext"></param>
protected override void Initialize(RequestContext requestContext)
{
base.Initialize(requestContext);
WebSite = ConfigHelper.LoadConfig <WebSiteModel>(ConfigFileEnum.SiteConfig);
WorkContext.SessionId = Session.SessionID;
WorkContext.IsHttpAjax = WebHelper.IsAjax();
WorkContext.IsHttpPost = WebHelper.IsPost();
WorkContext.IP = WebHelper.GetIP();
WorkContext.Url = WebHelper.GetUrl();
WorkContext.UrlReferrer = WebHelper.GetUrlReferrer();
WorkContext.AjaxStringEntity = new UnifiedResultEntity <string>();
WorkContext.AjaxStringEntity.msg = 0; //默认错误
WorkContext.AjaxStringEntity.total = 0;
//设置当前控制器类名
WorkContext.Controller = RouteData.Values["controller"].ToString().ToLower();
//设置当前动作方法名
WorkContext.Action = RouteData.Values["action"].ToString().ToLower();
WorkContext.PageKey = string.Format("{0}/{1}", WorkContext.Controller, WorkContext.Action).ToLower();
WorkContext.PageKeyCookie = ("admin" + WorkContext.Controller + WorkContext.Action).ToLower();
//用户
WorkContext.UserInfo = GetUserInfo();
}
public ActionResult Index(WebSiteModel model)
{
try
{
Configs.SetValue("SiteName", model.SiteName);
Configs.SetValue("SiteDomain", model.SiteDomain);
Configs.SetValue("MetaKey", model.MetaKey);
Configs.SetValue("MetaDescribe", model.MetaDescribe);
Configs.SetValue("MaxFileUpload", model.MaxFileUpload);
Configs.SetValue("CacheTime", model.HomeTitle);
Configs.SetValue("CacheTime", model.CacheTime);
Configs.SetValue("CopyRight", model.CopyRight);
Configs.SetValue("UploadFileType", model.UploadFileType);
}
catch (Exception ex)
{
ViewBag.Msg = "Error:" + ex.Message;
return(View(GetWebSiteInfo()));
}
ViewBag.Msg = "修改成功！";
return(View(GetWebSiteInfo()));
}
public SignModel BeginSign(string userName, string userPwd)
{
SignModel signModel = new SignModel();
WebSiteModel model = new WebSiteModel()
{
SiteDomin = "http://www.jobbole.com",
SiteName = "伯乐在线",
LoginUrl = "http://www.jobbole.com/wp-admin/admin-ajax.php",
LoginName = userName,
LoginPwd = userPwd,
SignRequestUrl = "http://www.jobbole.com/wp-admin/admin-ajax.php",
};
model.Cookie = GetCookie(model);
model.Result = PostLogin(model);
if (model.Result.Html.Contains("jb_result\":0"))//登陆成功
{
model.Cookie = model.Result.Cookie;
signModel.Msg = "登陆成功";
model.Result = PostSign(model);
if (model.Result.Html.Contains("jb_result\":0"))
{
signModel.Msg = "签到成功";
signModel.IsSuccess = true;
}
else if (model.Result.Html.Contains("jb_result\":-1"))
{
signModel.Msg = RegexHelper.GeMidStringValue(model.Result.Html, "jb_msg\":\"", "\"}").ToGB2312();
}
}
else
{
signModel.Msg = "登陆失败";
}
signModel.Msg = "[" + userName + "]" + signModel.Msg;
return signModel;
}
public List <WebSiteModel> ExtractData(string websiteContent)
{
List <string> listOfNumbers = File.ReadAllLines(_convertedNumbers).ToList();
List <string> listOfNotInterested = File.ReadAllLines(_notInterested).ToList();
List <WebSiteModel> outputList = new List <WebSiteModel>();
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(websiteContent);
var htmlBody = htmlDoc.DocumentNode.SelectSingleNode("//body");
var mainBoxNodes = htmlBody.Descendants("div").Where(node => node.GetAttributeValue("class", String.Empty).Contains("box-content company-row list-free")).ToList();
foreach (HtmlNode nodes in mainBoxNodes)
{
WebSiteModel companyDetails = new WebSiteModel();
try
{
//company Name
companyDetails.Company = nodes.Descendants("h2").Where(node => node.GetAttributeValue("class", String.Empty).Equals("company-name"))?.FirstOrDefault().InnerText;
}
catch (Exception)
{
companyDetails.Company = "BRAK";
}
try
{
//company Phone
companyDetails.PhoneNumber = nodes.Descendants("a").Where(node => node.GetAttributeValue("href", String.Empty).Equals("javascript:;"))?.FirstOrDefault().Attributes["data-phone"].Value;
if (listOfNumbers.Contains(companyDetails.PhoneNumber))
{
companyDetails.PhoneNumber = "CONVERTED";
}
if (listOfNotInterested.Contains(companyDetails.PhoneNumber))
{
companyDetails.PhoneNumber = "NOT INTERESTED";
}
}
catch (Exception)
{
companyDetails.PhoneNumber = "BRAK";
}
try
{
//company Email
HtmlNode emailNode = nodes.Descendants("div").Where(node => node.GetAttributeValue("class", String.Empty).Equals("call-cell call--email"))?.FirstOrDefault();
companyDetails.Email = emailNode.Descendants("span")?.FirstOrDefault().Attributes["title"].Value;
}
catch (Exception)
{
companyDetails.Email = "BRAK";
}
outputList.Add(companyDetails);
}
return(outputList);
}
public ActionResult InsertWebSite()
{
WebSiteModel u = new WebSiteModel();
return(View(u));
}
/// <summary>
/// 提交签到请求
/// </summary>
/// <param name="model"></param>
private HttpResult PostSign(WebSiteModel model)
{
var result = http.GetHtml(new HttpItem()
{
URL = model.SignRequestUrl,
Cookie = model.Cookie,
Referer = model.SiteDomin,
Method = "Post",
Postdata = "action=get_login_point",
Timeout = 100000,//连接超时时间 可选项默认为100000
ReadWriteTimeout = 30000,//写入Post数据超时时间 可选项默认为30000
//UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36",//用户的浏览器类型，版本，操作系统 可选项有默认值
Accept = "application/json, text/javascript, */*",// 可选项有默认值
ContentType = "application/x-www-form-urlencoded",//返回类型 可选项有默认值
});
return result;
}
public CustomActionAttribute()
{
SiteConfig = ConfigHelper.LoadConfig <WebSiteModel>(ConfigFileEnum.SiteConfig);
}
public void GetSelectedContact(WebSiteModel model)
{
SelectedCredentialsFromWeb = model;
}
public List <WebSiteModel> ExtractData(string websiteContent)
{
List <string> listOfNumbers = File.ReadAllLines(_convertedNumbers).ToList();
List <string> listOfNotInterested = File.ReadAllLines(_notInterested).ToList();
List <WebSiteModel> outputList = new List <WebSiteModel>();
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(websiteContent);
var htmlBody = htmlDoc.DocumentNode.SelectSingleNode("//body");
var mainBoxNodes = htmlBody.Descendants("div").Where(node => node.GetAttributeValue("class", String.Empty).Contains("row bottom-content px-3")).ToList();
foreach (HtmlNode nodes in mainBoxNodes)
{
WebSiteModel companyDetails = new WebSiteModel();
try
{
//company Name
string linkToClean = nodes.Descendants("a").Where(node => node.GetAttributeValue("class", String.Empty).Equals("icon-telephone addax addax-cs_hl_phonenumber_click"))?.FirstOrDefault().Attributes["href"].Value;
string[] splitLink = linkToClean.Split('/');
companyDetails.Company = splitLink.Last().Replace(".html", string.Empty);
}
catch (Exception)
{
companyDetails.Company = "BRAK";
}
try
{
//company Phone
companyDetails.PhoneNumber = nodes.Descendants("a").Where(node => node.GetAttributeValue("class", String.Empty).Contains("icon-telephone addax addax-cs_hl_phonenumber_click"))?.FirstOrDefault().Attributes["title"].Value;
if (listOfNumbers.Contains(companyDetails.PhoneNumber))
{
companyDetails.PhoneNumber = "CONVERTED";
}
if (listOfNotInterested.Contains(companyDetails.PhoneNumber))
{
companyDetails.PhoneNumber = "NOT INTERESTED";
}
}
catch (Exception)
{
companyDetails.PhoneNumber = "BRAK";
}
try
{
//company Email
companyDetails.Email = nodes.Descendants("div").Where(node => node.GetAttributeValue("data-toggle", String.Empty).Contains("tooltip"))?.FirstOrDefault().Attributes["title"].Value.Trim();
}
catch (Exception)
{
companyDetails.Email = "BRAK";
}
outputList.Add(companyDetails);
}
return(outputList);
}