/// <summary>
/// Should connections be captured for the given username?
/// Not capturing connections will prevent other profiles from being parsed from this one.
/// </summary>
/// <param name="userName">User Name to capture connections</param>
/// <returns>true if connections should be captured</returns>
public bool CanCaptureConnections(String userName)
{
if (!string.IsNullOrEmpty(LocationCriteria))
{
String location = CrawlUtil.GetUserLocation(userName);
if (AllowEmptyLocations)
{
return(String.IsNullOrEmpty(location) || (location.ToLower().Contains(LocationCriteria)));
}
else
{
return(!String.IsNullOrEmpty(location) && (location.ToLower().Contains(LocationCriteria)));
}
}
else
{
return(true);
}
}
/// <summary>
/// Start crawl.
/// </summary>
public void Crawl()
{
//Ensure HTTPS will work correctly
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
List <ProfileLocationRecord> records = new List <ProfileLocationRecord>();
using (Logger log = new Logger(StoreDirectory, "locationDownload"))
{
//Load existing usernames
try
{
records = JsonConvert.DeserializeObject <List <ProfileLocationRecord> >(File.ReadAllText(Path.Combine(StoreDirectory, "locations.json")));
if (records == null)
{
log.Log("Could not load locations file.");
records = new List <ProfileLocationRecord>();
}
}
catch (Exception e)
{
log.Log("Could not load locations file.");
records = new List <ProfileLocationRecord>();
}
String[] profileFiles = Directory.GetFiles(StoreDirectory, @"*.profile.json", SearchOption.AllDirectories);
foreach (String profileFileName in profileFiles)
{
if (!String.IsNullOrEmpty(profileFileName))
{
try
{
Profile parentProfile = JsonConvert.DeserializeObject <Profile>(File.ReadAllText(profileFileName));
if (parentProfile == null)
{
log.Log(String.Format(@"Empty Profile: {0}", profileFileName));
}
else
{
log.Log(String.Format("Processing {0}", parentProfile?.UserName));
if (records.Where(x => String.Equals(x.UserName, parentProfile.UserName)).FirstOrDefault() == null)
{
//Add parent record
records.Add(new ProfileLocationRecord()
{
UserName = parentProfile.UserName,
PersonalName = parentProfile.PersonalName,
Location = parentProfile.LocationDescription,
ConnectedFromLocation = null,
Error = null,
});
log.Log(String.Format("Added {0}", parentProfile.UserName));
}
//Process all connection records
int connectionCount = 0;
int connectionTotal = parentProfile.Connections != null ? parentProfile.Connections.Count : 0;
foreach (ConnectionEntry connection in parentProfile.Connections)
{
connectionCount++;
try
{
//Check if exists
if (records.Where(x => String.Equals(x.UserName, connection.UserName)).FirstOrDefault() == null)
{
//Get the location, then add the record
String locationDescription = CrawlUtil.GetUserLocation(connection.UserName);
Thread.Sleep(20);
records.Add(new ProfileLocationRecord()
{
UserName = connection.UserName,
PersonalName = connection.PersonalName,
Location = locationDescription,
ConnectedFromLocation = parentProfile.LocationDescription,
Error = null,
});
log.Log(String.Format(@"({1}/{2})Added {0}", connection.UserName, connectionCount, connectionTotal));
}
else
{
//log.Log(String.Format("({1}/{2})Skipped {0}", connection.UserName, connectionCount, connectionTotal));
}
}
catch (Exception e)
{
log.Log(String.Format("Error: {0}", e?.Message));
}
}
}
}
catch (Exception e)
{
log.Log(String.Format(@"Error loading profile: {0}", profileFileName));
return;
}
}
//Update location file
try
{
File.WriteAllText(
Path.Combine(StoreDirectory, "locations.json"),
JsonConvert.SerializeObject(records, Formatting.Indented));
}
catch (Exception e)
{
log.Log(String.Format(@"Error Saving Locations: {0}", e?.Message));
return;
}
}
}
}