public async Task <GithubProfile> GetNewGithubProfile(string link)
{
GithubProfile profile;
try
{
using (var db = new MonitoringEntities())
{
driver.Navigate().GoToUrl(link);
Thread.Sleep(2000);
profile = GetGithubProfileSelenium(driver.PageSource, link);
if (profile != null)
{
var repositories = GetRepositories(link);
profile.GithubRepositories = repositories;
if (githubUrls.Contains(link))
{
ProfileUpdate(profile, link, db);
}
else
{
db.GithubProfiles.Add(profile);
}
await db.SaveChangesAsync();
}
}
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
return(null);
}
return(profile);
}
public override void OnActionExecuting(ActionExecutingContext filterContext)
{
MonitoringEntities db = new MonitoringEntities();
var request = filterContext.HttpContext.Request;
int _id = (db.log_event.Count() > 0) ? (db.log_event.Max(p => p.id) + 1) : 1;
log_event log_event = new log_event()
{
id = _id,
title = request.RawUrl,
date = DateTime.Now,
kind_event_id = _kind_event,
IP_adress_object = request.ServerVariables["HTTP_X_FORWARED_FOR"] ?? request.UserHostAddress,
description = request.RequestType
//login = filterContext.HttpContext.User.Identity.Name.ToString(),
};
if (Convert.ToInt32(filterContext.HttpContext.Session["user_id"]) != 0)
{
log_event.author_id = Convert.ToInt32(filterContext.HttpContext.Session["user_id"]);
}
db.log_event.Add(log_event);
db.SaveChanges();
base.OnActionExecuting(filterContext);
}
public void LoadProxy()
{
using (MonitoringEntities db = new MonitoringEntities())
{
_proxies = db.Proxies.Where(x => x.Type.Contains("HTTP")).ToList();
}
}
/// <summary>
/// Get and save in DB Github profile using selenium
/// </summary>
public static void GetGithubProfileSelenium()
{
try
{
using (var db = new MonitoringEntities())
{
var profileFirefox = new FirefoxProfile(FirefoxProfilePath);
var driverProfile = new FirefoxDriver(new FirefoxOptions {
Profile = profileFirefox
});
foreach (var link in db.GithubProfiles.Select(x => x.Url).ToList())
{
driverProfile.Navigate().GoToUrl(link);
Thread.Sleep(2000);
var profile = GetGithubProfileSelenium(driverProfile.PageSource, link);
if (profile != null)
{
profile.LastUpdate = DateTime.Now;
db.GithubProfiles.AddOrUpdate(profile);
}
}
}
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
//异步处理报警逻辑(汉王数据接入使用)
public async void AsyncHandleAlarmPolicies1(string deviceId, string air, string curValue, DateTime now)
{
RetDeviceInfo deviceInfo = new RetDeviceInfo();
RetDeviceItemInfo itemInfo = new RetDeviceItemInfo();
using (MonitoringEntities device = new MonitoringEntities())
{
try
{
var searchDevice = device.D_Devices.Where(s => s.DeviceLabel == deviceId).FirstOrDefault();
if (searchDevice != null)
{
deviceInfo.ID = searchDevice.ID;
deviceInfo.Name = searchDevice.Name;
deviceInfo.DeviceLabel = searchDevice.DeviceLabel;
var searchDeviceItem = device.D_DevicesItem.Where(s => s.DeviceID == searchDevice.ID && s.PropertyLabel == air).FirstOrDefault();
if (searchDeviceItem != null)
{
itemInfo.ID = searchDeviceItem.ID;
itemInfo.Name = itemInfo.Name;
itemInfo.PropertyLabel = itemInfo.PropertyLabel;
}
}
}
catch (Exception e)
{
log.ErrorFormat("内部错误：{0},{1}", e.Message, e.StackTrace);
}
var result = await HandleAlarmPolicies1(deviceInfo, itemInfo, curValue, now);
}
}
/// <summary>
/// Get all linkedin profiles using links in db or links from param
/// </summary>
/// <param name="links">List of linkedin links</param>
/// <returns></returns>
public List <string> GetAllLinkedinProfiles(List <string> links = null)
{
var jsons = new List <string>();
if (links != null)
{
_linkedinLinks = links;
}
foreach (var linkedinLink in _linkedinLinks)
{
try
{
_driver.Navigate().GoToUrl(linkedinLink);
Thread.Sleep(4000);
Scroll(_driver);
var linkedinProfile = GetProfile(_driver.PageSource);
Thread.Sleep(1000);
if (linkedinProfile != null)
{
try
{
var username = linkedinLink.Split(new[] { "in/" }, StringSplitOptions.RemoveEmptyEntries).LastOrDefault()?.TrimEnd('/');
linkedinProfile.Username = username;
using (MonitoringEntities monitoringEntities = new MonitoringEntities())
{
var user = monitoringEntities.LinkedinProfiles.Where(x => x.Username == username).ToList().FirstOrDefault();
if (user != null)
{
UpdateLinkedinProfile(linkedinProfile, user);
//monitoringEntities.Entry(user).State = EntityState.Modified;
//monitoringEntities.LinkedinProfiles.AddOrUpdate(user);
}
else
{
linkedinProfile.LastUpdate = DateTime.Now;
monitoringEntities.LinkedinProfiles.Add(linkedinProfile);
}
monitoringEntities.SaveChanges();
Logger.Info($"So good for profile {username}");
}
jsons.Add(JsonConvert.SerializeObject(linkedinProfile, Formatting.Indented, new JsonSerializerSettings()
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
}));
}
catch (Exception e)
{
Logger.Error(e, $"{MethodBase.GetCurrentMethod().Name} -- {linkedinLink}");
}
}
}
catch (Exception e)
{
Logger.Error(e, $"{MethodBase.GetCurrentMethod().Name} -- {linkedinLink}");
}
}
return(jsons);
}
/// <summary>
/// Get and save in DB all Github profile repositories
/// </summary>
public static void GetRepositories()
{
try
{
using (var db = new MonitoringEntities())
{
var profiles = db.GithubProfiles.ToList();
foreach (var profile in profiles)
{
Thread.Sleep(5000);
var repositories = GetRepositories(profile.Url);
if (repositories != null)
{
profile.GithubRepositories = repositories;
profile.LastUpdate = DateTime.Now;
db.SaveChanges();
}
}
}
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
protected override void OnStart(string[] args)
{
HidemeParser _hidemeParser = new HidemeParser();
var proxies = _hidemeParser.GetProxy(cookie).Result;
using (MonitoringEntities db = new MonitoringEntities())
{
db.Database.ExecuteSqlCommand("delete from proxy");
db.Proxies.AddRange(proxies);
db.SaveChanges();
}
}
private void ProfileUpdate(GithubProfile profile, string url, MonitoringEntities db)
{
try
{
var profileInDb = db.GithubProfiles.FirstOrDefault(x => x.Url == url);
if (profileInDb == null)
{
return;
}
if (!string.IsNullOrEmpty(profile.Bio))
{
profileInDb.Bio = profile.Bio;
}
if (!string.IsNullOrEmpty(profile.BlogOrWebsite))
{
profileInDb.BlogOrWebsite = profile.BlogOrWebsite;
}
if (!string.IsNullOrEmpty(profile.Company))
{
profileInDb.Company = profile.Company;
}
if (!string.IsNullOrEmpty(profile.Email))
{
profileInDb.Email = profile.Email;
}
if (!string.IsNullOrEmpty(profile.Name))
{
profileInDb.Name = profile.Name;
}
if (!string.IsNullOrEmpty(profile.Location))
{
profileInDb.Location = profile.Location;
}
if (!string.IsNullOrEmpty(profile.ImageUrl))
{
profileInDb.ImageUrl = profile.ImageUrl;
}
if (profileInDb.StarsCount != 0)
{
profileInDb.StarsCount = profile.StarsCount;
}
profileInDb.LastUpdate = DateTime.Now;
db.GithubProfiles.AddOrUpdate(profileInDb);
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
public void LoadProxy()
{
try
{
using (var db = new MonitoringEntities())
{
_proxies = db.Proxies.Where(x => x.Type == "HTTPS").ToList();
}
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
public override string[] GetRolesForUser(string username)
{
string[] roles = new string[] { };
using (MonitoringEntities db = new MonitoringEntities())
{
int role = Convert.ToInt32(HttpContext.Current.Session["role"]);
User user = db.User.Include(u => u.Role).FirstOrDefault(u => u.email == username && u.role_id == role);
if (user != null && user.Role != null)
{
HttpContext.Current.Session["message"] = user.countMessage();
HttpContext.Current.Session["notification"] = user.countNotification();
roles = new string[] { user.Role.name };
}
return(roles);
}
}
/// <summary>
/// Ctor
/// </summary>
public Linkedin()
{
try
{
using (var entities = new MonitoringEntities())
{
var links = entities.LinkedinProfiles.OrderBy(x => x.LastUpdate).Select(x => x.Username).ToList().Select(x => $"https://www.linkedin.com/in/{x}").ToList();
_linkedinLinks = links;
_driver = new FirefoxDriver();
}
Login();
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
public override bool IsUserInRole(string username, string roleName)
{
using (MonitoringEntities db = new MonitoringEntities())
{
// Получаем пользователя
User user = db.User.Include(u => u.Role).FirstOrDefault(u => u.email == username);
if (user != null && user.Role != null && user.Role.name == roleName)
{
return(true);
}
else
{
return(false);
}
}
}
static void Main(string[] args)
{
var textDetection = new TextDetection();
string vanik1 = "vanik hakobyan";
string vanik2 = "van hakobyan";
var similarity = textDetection.GetSimilarity(vanik1, vanik2, AlgorithmTypes.SmithWatermanGotohWindowedAffine);
using (var db = new MonitoringEntities())
{
var githubLinkedinCrossTables = db.GithubLinkedinCrossTables.ToList();
var gList = githubLinkedinCrossTables.Select(a => a.GithubUserId);
var lList = githubLinkedinCrossTables.Select(a => a.LinkedinUserId);
var githubProfiles = db.GithubProfiles.Where(x => !gList.Contains(x.Id)).ToList();
var linkedinProfiles = db.LinkedinProfiles.Where(x => !lList.Contains(x.Id)).ToList();
foreach (var githubProfile in githubProfiles)
{
if (string.IsNullOrEmpty(githubProfile.Name) || string.IsNullOrEmpty(githubProfile.UserName))
{
continue;
}
foreach (var linkedinProfile in linkedinProfiles)
{
if (string.IsNullOrEmpty(linkedinProfile.FullName) || string.IsNullOrEmpty(linkedinProfile.Username))
{
continue;
}
var similarityByName = textDetection.GetSimilarity(githubProfile.Name, linkedinProfile.FullName, AlgorithmTypes.OverlapCoefficient);
var similarityByUserName = textDetection.GetSimilarity(githubProfile.UserName, linkedinProfile.Username, AlgorithmTypes.OverlapCoefficient);
if (similarityByName > 0.9 && similarityByUserName > 0.9)
{
if (db.GithubLinkedinCrossTables.FirstOrDefault(x => x.GithubUserId == githubProfile.Id) == null)
{
db.GithubLinkedinCrossTables.Add(new GithubLinkedinCrossTable()
{
GithubUserId = githubProfile.Id, LinkedinUserId = linkedinProfile.Id
});
db.SaveChanges();
}
break;
}
}
}
}
}
public List <string> LoadUrlInDb()
{
List <string> urls;
using (var db = new MonitoringEntities())
{
try
{
urls = db.GithubProfiles.Select(x => x.Url).ToList();
githubUrls = urls;
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
urls = new List <string>();
githubUrls = new List <string>();
}
}
return(urls);
}
public LastValue2 GetLastValue(int id)
{
Entities = new MonitoringEntities();
LastValue lastValue = Entities.LastValues.FirstOrDefault(x => x.Id == id);
LastValue2 result = new LastValue2();
if (lastValue != null)
{
result.Id = id;
result.Date = lastValue.Date;
if (lastValue.Date != null)
{
result.PersianDateString = ((DateTime)lastValue.Date).ToPersianString();
}
result.Value = lastValue.Value;
result.HasError = false;
}
return(result);
}
/// <summary>
/// Scrape and save in DB proxies
/// <param name="cookie">Hide me request cookie</param>
/// </summary>
public static void ScrapProxyFromHideMe(string cookie = null)
{
var pr = new HidemeParser();
try
{
var proxies = pr.GetProxy(cookie).Result.ToList();
using (var db = new MonitoringEntities())
{
foreach (var proxy in proxies)
{
db.Proxies.Add(new Proxy {
Country = proxy.Country, Port = proxy.Port, Type = proxy.Type, Ip = proxy.Ip
});
}
db.SaveChanges();
}
}
catch (Exception e)
{
Logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
public void GetCompanies()
{
foreach (var link in Links)
{
try
{
CultureInfo cultureInfo = Thread.CurrentThread.CurrentCulture;
TextInfo textInfo = cultureInfo.TextInfo;
using (MonitoringEntities context = new MonitoringEntities())
{
var cName = link.Split('/').LastOrDefault()?.Replace('-', ' ');
var company = context.Companies.FirstOrDefault(c => c.Name.ToLower() == cName);
if (company is null)
{
continue; company = new Company {
Name = textInfo.ToTitleCase(cName)
};
}
var companyHtmlContent = SendGetRequest(link).Result;
HtmlDocument document = new HtmlDocument();
document.LoadHtml(companyHtmlContent);
//var headerInfoNode = document.DocumentNode.SelectSingleNode(".//div[@class='header-info accordion-style']");
//var companyDetails = document.DocumentNode.SelectSingleNode(".//div[@id='company-details']");
//var jobsListNode =document.DocumentNode.SelectSingleNode(".//div[@class='accordion-style clearfix company-jobs-list']");
//var contactDetails = document.DocumentNode.SelectSingleNode(".//div[@id ='company-contact-details']");
//if (headerInfoNode != null) GetHeaderInfo(company, headerInfoNode);
//if (companyDetails != null) GetDetails(company, companyDetails);
//if (contactDetails != null) GetContact(company, contactDetails);
var jobList = document.DocumentNode.SelectNodes(".//a[@class='hs_company_more_media_btn']")
?.Select(x => x.GetAttributeValue("href", ""))?.ToList();
if (company.Jobs.Count > 0)
{
context.Jobs.RemoveRange(company.Jobs);
context.SaveChanges();
}
if (jobList != null)
{
var jobs = GetJobs(jobList);
company.Jobs = jobs;
}
//context.Companies.AddOrUpdate(company);
context.SaveChanges();
}
}
catch (Exception e)
{
_logger.Error(e, MethodBase.GetCurrentMethod().Name);
}
}
}
public void UpdateGithubProfilesInDb()
{
using (MonitoringEntities db = new MonitoringEntities())
{
var githubProfiles = db.GithubProfiles.ToList();
var allFromApi = GetAllFromApiAsync();
for (var i = 0; i < allFromApi.Count; i++)
{
var githubApiModel = allFromApi[i];
try
{
var userJson = SendGetRequest(githubApiModel.url);
var userModel = JsonConvert.DeserializeObject <GithubApiUserModel>(userJson);
var updateProfile = githubProfiles.FirstOrDefault(x => x.UserName == userModel.login);
if (updateProfile is null)
{
var newGithubMember = new GithubProfile
{
UserName = userModel.login,
Email = userModel.email as string,
Company = userModel.company as string,
Location = userModel.location,
ImageUrl = userModel.avatar_url,
BlogOrWebsite = userModel.blog,
Name = userModel.name,
Bio = userModel.bio as string,
Url = userModel.html_url,
LastUpdate = DateTime.Now
};
db.GithubProfiles.Add(newGithubMember);
}
else
{
if (userModel.email != null)
{
updateProfile.Email = userModel.email as string;
}
if (userModel.company != null)
{
updateProfile.Company = userModel.company as string;
}
if (userModel.location != null)
{
updateProfile.Location = userModel.location;
}
if (userModel.avatar_url != null)
{
updateProfile.ImageUrl = userModel.avatar_url;
}
if (userModel.blog != null)
{
updateProfile.BlogOrWebsite = userModel.blog;
}
if (userModel.name != null)
{
updateProfile.Name = userModel.name;
}
if (userModel.bio != null)
{
updateProfile.Bio = userModel.bio as string;
}
updateProfile.LastUpdate = DateTime.Now;
db.GithubProfiles.AddOrUpdate(updateProfile);
}
db.SaveChanges();
//Thread.Sleep(1500);
}
catch
{
Console.WriteLine(i);
}
}
}
}
