public static UsernameLinkModel ExtractUsernamesFromSource(DBLinkModel target, string source)
{
//Unauthenticated pattern
//string pattern = @"(?<=<a class=""_42ft _4jy0 _4jy3 _517h _51sy"" role=""button"" href=""https:\/\/www.facebook.com\/).*?(?=\/photos"")";
//Authenticated pattern
string pattern = FacebookRegexConfiguration.ExtractUserNamesFromProfilePageRegex;
string pattern2 = "(?<=class=\"_2ial).*(https:// www.facebook.*)(?=\" data-testid=\"serp_result)";
List <LinkModel> usernames = new List <LinkModel>();
MatchCollection matches = Regex.Matches(source, pattern);
if (matches.Count == 0)
{
matches = Regex.Matches(source, pattern2);
}
foreach (Match m in matches)
{
usernames.Add(new LinkModel()
{
ProfileLink = m.Value, ConfidenceScore = 0
});
}
UsernameLinkModel linkModel = new UsernameLinkModel()
{
id = target.Id,
name = target.firstname + " " + target.lastname,
usernames = usernames
};
Console.WriteLine("Links Made: " + usernames.Count);
return(linkModel);
}
public static string FormatQueryURL(DBLinkModel target)
{
StringBuilder builder = new StringBuilder();
builder.Append(FacebookRegexConfiguration.FacebookQueryURL + target.firstname + "+" + target.lastname + "+" + target.state);
return(builder.ToString());
}
public static double GetConfidenceLevel(DBLinkModel targetModel, string source)
{
double confidenceLevel = 0.0;
string realName = FacebookStuff.GetRealNameFromProfilePage(source);
List <string> details = FacebookStuff.GetIntroFromAuthenticatedProfilePage(source);
List <string> texts = FacebookStuff.GetTextFromDetailsAuthenticated(details);
confidenceLevel += ScoreRealName(targetModel, realName);
confidenceLevel += ScoreCityState(targetModel, texts);
return(confidenceLevel);
}
private static double ScoreCityState(DBLinkModel personModel, List <string> details)
{
double returnScore = 0.0;
string personState = personModel.state.ToLower();
string personCity = personModel.city.ToLower();
bool cityFound = false;
bool stateFound = false;
int stateKeywordCount = 0;
int cityKeywordCount = 0;
foreach (string detail in details)
{
if (detail.ToLower().Contains(personState))
{
stateFound = true;
stateKeywordCount++;
}
}
foreach (string detail in details)
{
if (detail.ToLower().Contains(personCity))
{
cityFound = true;
stateKeywordCount++;
}
}
if (cityFound && stateFound)
{
returnScore += .7;
}
else if (cityFound && !stateFound)
{
returnScore += .4;
}
else if (stateFound && !cityFound)
{
returnScore += .1;
}
returnScore += (stateKeywordCount * .01) + (cityKeywordCount * 2);
return(returnScore);
}
public static async Task <List <DBLinkModel> > RunQuery(string fname, string lname, string city)
{
List <DBLinkModel> targets = new List <DBLinkModel>();
try
{
InitializeDB();
string _sqlCommand = ("select distinct * from (select LAST_NAME, FIRST_NAME, DATE_OF_BIRTH, RESIDENTIAL_ADDRESS1, RESIDENTIAL_CITY from " + ApplicationConfiguration.DatabaseTableName + " where RESIDENTIAL_CITY = @city AND FIRST_NAME = @fname AND LAST_NAME = @lname) ss");
await _connection.OpenAsync();
SqlCommand command = new SqlCommand(_sqlCommand, _connection);
command.Parameters.AddWithValue("@fname", fname);
command.Parameters.AddWithValue("@lname", lname);
command.Parameters.AddWithValue("@city", city);
Console.WriteLine("Querying Database for users...");
SqlDataReader dataReader = await command.ExecuteReaderAsync();
int count = 0;
while (dataReader.Read())
{
count++;
DBLinkModel temp = new DBLinkModel()
{
Id = Guid.NewGuid(),
lastname = dataReader.GetValue(0).ToString(),
firstname = dataReader.GetValue(1).ToString(),
dob = dataReader.GetValue(2).ToString(),
address = dataReader.GetValue(3).ToString(),
city = dataReader.GetValue(4).ToString(),
state = ApplicationConfiguration.DatabaseState
};
targets.Add(temp);
}
dataReader.Close();
command.Dispose();
_connection.Close();
Console.WriteLine("ROWS: " + count);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message); /**nom nom nom **/
}
return(targets);
}
private static double ScoreRealName(DBLinkModel personModel, string realname)
{
double returnScore = 0.0;
string fname = personModel.firstname.ToLower();
string lname = personModel.lastname.ToLower();
string[] names = realname.Split(' ');
foreach (string name in names)
{
if (name.ToLower().Contains(fname))
{
returnScore += .15;
}
if (name.ToLower().Contains(lname))
{
returnScore += .15;
}
}
return(returnScore);
}
public void ConvertToPersonModelExecuted(object args)
{
PersonModel personModel = new PersonModel();
FacebookLinkModel facebookLinkModel = SelectedFacebookLinkModel;
personModel.ImageUrls.Add(facebookLinkModel.ProfileImage);
foreach (string job in facebookLinkModel.Jobs)
{
personModel.Jobs.Add(job);
}
personModel.Names.Add(facebookLinkModel.DisplayName);
personModel.Names.Add(facebookLinkModel.FirstName + " " + facebookLinkModel.LastName);
if (facebookLinkModel.PossibleLinks != null && facebookLinkModel.PossibleLinks.Count > 0)
{
DBLinkModel dblink = facebookLinkModel.PossibleLinks.FirstOrDefault();
personModel.Names.Add(dblink.FormatedName);
personModel.Dobs.Add(dblink.dob);
personModel.Addresses.Add(dblink.address);
}
foreach (Person pipLink in facebookLinkModel.PiplLinks)
{
if (!pipLink.Selected)
{
continue;
}
if (pipLink.dob != null)
{
personModel.Dobs.Add(pipLink.dob.date_range.start);
}
foreach (Name name in pipLink.names)
{
personModel.Names.Add(name.display);
}
foreach (Address address in pipLink.addresses)
{
personModel.Addresses.Add(address.display);
}
foreach (Job job in pipLink.jobs)
{
personModel.Jobs.Add(job.display);
}
foreach (Phone phone in pipLink.phones)
{
personModel.PhoneNumbers.Add(phone.display);
}
foreach (Image image in pipLink.images)
{
personModel.ImageUrls.Add(image.url);
}
foreach (Education education in pipLink.educations)
{
personModel.Schools.Add(education.school + " - " + education.degree);
}
foreach (string detail in facebookLinkModel.ParsedDetails)
{
personModel.Details.Add(detail);
}
}
MasterTargetListViewModel.AddTarget(personModel);
}
public void ScanFacebookExecuted(object param)
{
// Okay I apologize in advance for this shit.... Read on if you dare...
//
// Purpose of this block of code:
// To scrape facebook as fast as possible, asynchronously.
//
// How it's achieved:
// Parallel foreach loops iterating over urls and async web requests
//
// Why this is garbage:
// Parallel foreach is used to iterate over the urls as fast as possible. However, I'm using
// async web requests inside of the parallel loops. Unfortunately this breaks C# apparently.
// When you await an async request inside of a parallel foreach, the parallel loop is no longer
// blocking and exits immediately. This means that that part at the bottom, the part that updates the
// UI, will run prior to the results being returned.... Yeah... fml
//
// How I worked around this:
// I create a List<Object>. When a request goes out we add one to the list. When a response comes back
// we remove one from the list. At the base of the function we have an infinite non-blocking loop that
// waits for the requests count to equal zero. When it's zero it sorts our list based on confidence score
// and updates the UI.
//
// Problems with this method:
// Currently I'm not accounting for failed requests.... I have no idea what would happen. Probably nothing good.
// Like seriously... I don't expect the application to suddenly get better if a request fails... Fix this...
List <Object> requests = new List <Object>();
List <UsernameLinkModel> tempLinks = new List <UsernameLinkModel>();
try
{
IEnumerable <DBLinkModel> tmpCache = DatabaseSearchResults.Where(pm => pm.IsSelected);
foreach (var item in tmpCache)
{
var temp = TargetLinks.Where(tl => tl.id == item.Id && tl.usernames.Count > 0);
if (temp.Count() > 0)
{
tempLinks.Add(temp.FirstOrDefault());
tmpCache = tmpCache.Where(tl => tl.Id != item.Id);
}
}
TargetLinks.Clear();
TopTargets.Clear();
object outerLock = new object();
object innerLock = new object();
// First we run over our list of voter db targets
Parallel.ForEach(tmpCache, async pm =>
{
DBLinkModel target = pm;
Console.WriteLine("Running search on target: " + target.firstname + " " + target.lastname);
lock (outerLock)
{
requests.Add(new Object());
}
// foreach voter db target, try and see if there are any facebook users in ohio with that name
string source = await ExternalBrowser.CallExternalBrowser(FacebookStuff.FormatQueryURL(target), true);
// lock adding and removing requests, because you know, threads, shared resources, fml
lock (outerLock)
{
requests.RemoveAt(requests.Count - 1);
}
// Link made
TargetLinks.Add(FacebookStuff.ExtractUsernamesFromSource(target, source));
if (requests.Count == 0)
{
// For each link made
Parallel.ForEach(TargetLinks, targetLink =>
{
// Run over every possible user and score it
Parallel.ForEach(targetLink.usernames, async username =>
{
lock (innerLock)
{
requests.Add(new Object());
}
// Get the source for this possible facebook match
string profileSource = await ExternalBrowser.CallExternalBrowser(username.ProfileLink);
lock (innerLock)
{
requests.RemoveAt(requests.Count - 1);
}
// Parse and score the source
username.ConfidenceScore = RelationshipStuff.GetConfidenceLevel(SelectedPerson, profileSource);
username.UserModelLinkId = targetLink.id;
username.FullName = targetLink.name;
// wait until alllllllll the requests are done. If you're curious why in 2017 I need to do this, read the
// above giant comment block
while (requests.Count > 0)
{
System.Windows.Forms.Application.DoEvents();
}
Console.WriteLine("SCAN COMPLETED AT: " + DateTime.Now.ToString());
Sort();
});
});
}
});
}
catch (Exception err) { Console.WriteLine(err.Message); /*Nom nom nom*/ }
}
public static async Task <List <DBLinkModel> > RunQuery(string fname, string lname, DateTime dob, bool equal, bool after,
bool before, string streetAddress, string city, string zip)
{
List <DBLinkModel> targets = new List <DBLinkModel>();
try
{
InitializeDB();
string _sqlCommand = buildQuery(fname, lname, dob, equal, after, before, streetAddress, city, zip);
_connection.Open();
SqlCommand command = new SqlCommand(_sqlCommand, _connection);
if (!String.IsNullOrEmpty(fname))
{
command.Parameters.AddWithValue("@fname", fname);
}
if (!String.IsNullOrEmpty(lname))
{
command.Parameters.AddWithValue("@lname", lname);
}
if (dob != DateTime.MinValue)
{
command.Parameters.AddWithValue("@dob", dob.ToString("yyyy-MM-DD"));
}
if (!String.IsNullOrEmpty(streetAddress))
{
command.Parameters.AddWithValue("@raddress", streetAddress);
}
if (!String.IsNullOrEmpty(city))
{
command.Parameters.AddWithValue("@city", city);
}
if (!String.IsNullOrEmpty(zip))
{
command.Parameters.AddWithValue("@zip", zip);
}
Console.WriteLine("Querying Database for users...");
SqlDataReader dataReader = await command.ExecuteReaderAsync();
int count = 0;
while (dataReader.Read())
{
count++;
DBLinkModel temp = new DBLinkModel()
{
Id = Guid.NewGuid(),
lastname = dataReader.GetValue(0).ToString(),
firstname = dataReader.GetValue(1).ToString(),
dob = dataReader.GetValue(2).ToString(),
address = dataReader.GetValue(3).ToString(),
city = dataReader.GetValue(4).ToString(),
state = ApplicationConfiguration.DatabaseState
};
targets.Add(temp);
}
dataReader.Close();
command.Dispose();
_connection.Close();
Console.WriteLine("ROWS: " + count);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message); /**nom nom nom **/
}
return(targets);
}