private TextEntities ConvertToTextEntities(List <EnrichTextDataItem> enrichTextItems)
{
var result = new TextEntities();
foreach (EnrichTextDataItem item in enrichTextItems)
{
if (item.JsonData.IsNullOrEmpty())
{
continue;
}
if (item.Type == EnrchTextEntityType.Organization)
{
result.Organizations.Add(Json.Deserialize <OrganizationEntity>(item.JsonData));
}
else if (item.Type == EnrchTextEntityType.Address)
{
result.Addresses.Add(Json.Deserialize <AddressEntity>(item.JsonData));
}
else if (item.Type == EnrchTextEntityType.Communication)
{
result.Communications.Add(Json.Deserialize <CommunicationEntity>(item.JsonData));
}
else if (item.Type == EnrchTextEntityType.Contact)
{
result.Contacts.Add(Json.Deserialize <ContactEntity>(item.JsonData));
}
else if (item.Type == EnrchTextEntityType.Job)
{
result.Jobs.Add(Json.Deserialize <JobEntity>(item.JsonData));
}
}
return(result);
}
private void UpdateAccountIdentificationInfo(Guid enrchEmailDataId, TextEntities textEntities,
bool isNewEnrichedData)
{
HashSet <OrganizationEntity> organizations = textEntities.Organizations;
foreach (OrganizationEntity organizationEntity in organizations)
{
bool isAccountIdentified = false;
if (!isNewEnrichedData)
{
string organizationHash = organizationEntity.GetHash();
isAccountIdentified = CheckActualAccountIdentification(organizationHash, enrchEmailDataId);
}
if (isAccountIdentified)
{
continue;
}
IEnumerable <Guid> accountIds;
AccountIdentificationType identificationType = _enrichEntitySearchHelper
.IdentifyAccount(organizationEntity, textEntities, out accountIds);
if (!accountIds.IsNullOrEmpty())
{
Guid entityId = GetTextEntityId(enrchEmailDataId, isNewEnrichedData, organizationEntity);
if (entityId.IsEmpty())
{
continue;
}
_enrichEntitySearchHelper.AddFoundAccounts(entityId, accountIds, identificationType);
}
}
}
/// <summary>
/// Updates the account identification information.
/// </summary>
/// <param name="enrichTextItems">The enrich text items.</param>
/// <returns></returns>
public List <EnrichAccountItem> UpdateAccountIdentificationInfo(List <EnrichTextDataItem> enrichTextItems)
{
TextEntities textEntities = ConvertToTextEntities(enrichTextItems);
HashSet <OrganizationEntity> organizations = textEntities.Organizations;
var result = new List <EnrichAccountItem>();
foreach (OrganizationEntity organizationEntity in organizations)
{
IEnumerable <Guid> accountIds;
AccountIdentificationType identificationType = IdentifyAccount(organizationEntity, textEntities,
out accountIds);
var guids = accountIds != null?accountIds.ToList() : null;
if (!guids.IsNullOrEmpty())
{
Guid entityId = organizationEntity.Id;
AddFoundAccounts(entityId, guids, identificationType);
foreach (Guid id in guids)
{
result.Add(new EnrichAccountItem {
Id = id,
EnrchTextEntityId = entityId
});
}
}
}
return(result);
}
/// <summary>
/// Gets the emails.
/// </summary>
/// <param name="textEntities">The text entities.</param>
/// <param name="parentEntityId">The parent entity identifier.</param>
/// <returns></returns>
public static List <string> GetEmails(TextEntities textEntities, Guid parentEntityId)
{
List <string> emails = textEntities.Communications
.Where(entity => entity.Type == EnrchCommunicationType.Email && entity.ContactId == parentEntityId)
.Select(entity => entity.Value)
.ToList();
return(emails);
}
/// <summary>
/// Gets the phones.
/// </summary>
/// <param name="textEntities">The text entities.</param>
/// <param name="parentEntityId">The parent entity identifier.</param>
/// <returns></returns>
public static List <string> GetPhones(TextEntities textEntities, Guid parentEntityId)
{
List <string> phones = textEntities.Communications
.Where(entity => entity.IsPhone && entity.Type == EnrchCommunicationType.Mobile &&
entity.ContactId == parentEntityId)
.Select(entity => entity.Value)
.ToList();
return(phones);
}
private Entity CreateNewEnrichedData(TextEntities emailEntities)
{
EntitySchema enrchEmailDataSchema = _userConnection.EntitySchemaManager.GetInstanceByName("EnrchEmailData");
Entity newEnrichedData = enrchEmailDataSchema.CreateEntity(_userConnection);
newEnrichedData.SetDefColumnValues();
newEnrichedData.SetColumnValue("JsonData", JsonConvert.SerializeObject(emailEntities));
newEnrichedData.SetColumnValue("Hash", emailEntities.GetHash());
newEnrichedData.Save();
return(newEnrichedData);
}
public ActionResult Details(string id)
{
if (!string.IsNullOrEmpty(id))
{
var db = new TextEntities();
List <UserInfo> dbInfo = db.UserInfo.ToList();
UserInfoViewModel a = new UserInfoViewModel();
a.UserID = dbInfo[int.Parse(id)].UserID;
a.UserName = dbInfo[int.Parse(id)].UserName;
a.Phone = dbInfo[int.Parse(id)].Phone;
a.Email = dbInfo[int.Parse(id)].Email;
return(View(a));
}
return(View());
}
public ActionResult Delete(string id)
{
try
{
var db = new TextEntities();
List <UserInfo> dbInfo = db.UserInfo.ToList();
db.UserInfo.Remove(dbInfo[int.Parse(id)]);
db.SaveChanges();
return(RedirectToAction("UserList"));
}
catch
{
ViewData["Message"] = "刪除失敗了唷!";
return(View("UserList"));
}
}
public ActionResult Edit(string id)
{
var db = new TextEntities();
List <UserInfo> dbInfo = db.UserInfo.ToList();
var viewModelList = new List <UserInfoViewModel>();
var tmpInfo = new UserInfoViewModel()
{
UserID = dbInfo[int.Parse(id)].UserID,
UserName = dbInfo[int.Parse(id)].UserName,
Phone = dbInfo[int.Parse(id)].Phone,
Email = dbInfo[int.Parse(id)].Email,
};
viewModelList.Add(tmpInfo);
return(View(viewModelList));
}
private void ActualizeTextEntityHash(List <TextEntityData> textEntityDatas, Type type)
{
foreach (TextEntityData textEntityData in textEntityDatas)
{
var entity = (MiningEntity)JsonConvert.DeserializeObject(textEntityData.EteJsonData, type);
if (entity != null)
{
textEntityData.EteUpdatedHash = entity.GetHash();
}
TextEntities textEntities = JsonConvert.DeserializeObject <TextEntities>(textEntityData.EedJsonData);
if (textEntities != null)
{
textEntityData.EedUpdatedHash = textEntities.GetHash();
}
}
}
/// <summary>
/// Executes chain identification for account by name and communicatications.
/// </summary>
/// <param name="organizationEntity">The organization entity.</param>
/// <param name="textEntities">The text entities.</param>
/// <param name="accountIds">The account ids.</param>
/// <returns></returns>
public AccountIdentificationType IdentifyAccount(OrganizationEntity organizationEntity,
TextEntities textEntities, out IEnumerable <Guid> accountIds)
{
accountIds = _accountSearcher.SearchByName(organizationEntity.Value);
if (!accountIds.IsNullOrEmpty())
{
return(AccountIdentificationType.Name);
}
IEnumerable <CommunicationEntity> organizationCommunications =
from communication in textEntities.Communications
join contact in textEntities.Contacts on communication.ContactId equals contact.Id
where contact.OrganizationId == organizationEntity.Id
select communication;
return(IdentifyAccountByEte(organizationCommunications.ToList(), out accountIds));
}
private Guid FindPrimaryContact(TextEntities textEntities, Guid enrchEmailDataId, bool newContact)
{
ContactEntity primaryContact = textEntities.Contacts
.FirstOrDefault(contact => contact.IsFoundInLastEmail);
if (primaryContact == null)
{
return(Guid.Empty);
}
if (newContact)
{
return(primaryContact.Id);
}
else
{
string hash = primaryContact.GetHash();
return(GetTextEntityId(hash, enrchEmailDataId));
}
}
public ActionResult UserList()
{
var db = new TextEntities();
List <UserInfo> dbInfo = db.UserInfo.ToList();
var viewModelList = new List <UserInfoViewModel>();
foreach (UserInfo info in dbInfo)
{
var tmpInfo = new UserInfoViewModel()
{
UserID = info.UserID,
UserName = info.UserName,
Phone = info.Phone,
Email = info.Email,
};
viewModelList.Add(tmpInfo);
}
return(View(viewModelList));
}
public ActionResult Create(UserInfo info)
{
var db = new TextEntities();
List <UserInfo> dbInfo = db.UserInfo.ToList();
var newuserinfo = new UserInfo();
foreach (UserInfo theinfo in dbInfo)
{
if (info.UserID.Equals(theinfo.UserID))
{
ViewData["Message"] = "客戶編號重複，請更改編號！";
return(View());
}
}
newuserinfo = info;
db.UserInfo.Add(newuserinfo);
db.SaveChanges();
return(RedirectToAction("UserList"));
}
public ActionResult Details(string id, UserInfo info)
{
var db = new TextEntities();
List <UserInfo> dbInfo = db.UserInfo.ToList();
try
{
dbInfo[int.Parse(id)].UserID = info.UserID;
dbInfo[int.Parse(id)].UserName = info.UserName;
dbInfo[int.Parse(id)].Phone = info.Phone;
dbInfo[int.Parse(id)].Email = info.Email;
db.SaveChanges();
return(RedirectToAction("UserList"));
}
catch
{
ViewData["Message"] = "修改出錯了唷!";
return(RedirectToAction("Edit"));
}
}
private IEnumerable <Guid> FindDuplicateCommunications(Dictionary <Guid, Guid> identifiedContacts,
TextEntities textEntities)
{
List <Guid> duplicateCommunications = new List <Guid>();
IEnumerable <CommunicationEntity> minedCommunicationEntities = textEntities.Communications
.Where(minedCommunication => identifiedContacts.ContainsKey(minedCommunication.ContactId));
if (minedCommunicationEntities.IsEmpty())
{
return(duplicateCommunications);
}
Dictionary <Guid, HashSet <string> > phones;
Dictionary <Guid, HashSet <string> > others;
HashSet <Guid> contactIds = new HashSet <Guid>(identifiedContacts.Values);
LoadCommunications(contactIds, out phones, out others);
foreach (CommunicationEntity minedCommunication in minedCommunicationEntities)
{
Guid identifiedContact = identifiedContacts[minedCommunication.ContactId];
if (minedCommunication.IsPhone)
{
HashSet <string> knownContactPhones = phones[identifiedContact];
if (PhoneComparer.MatchAny(minedCommunication.Value, knownContactPhones))
{
duplicateCommunications.Add(minedCommunication.Id);
}
}
else
{
HashSet <string> knownOtherCommunications = others[identifiedContact];
if (knownOtherCommunications.Contains(minedCommunication.Value))
{
duplicateCommunications.Add(minedCommunication.Id);
}
}
}
return(duplicateCommunications);
}
private IEnumerable <Guid> FindDuplicateJobs(Dictionary <Guid, Guid> identifiedContacts,
TextEntities textEntities)
{
List <Guid> duplicateJobs = new List <Guid>();
IEnumerable <JobEntity> minedJobEntities = textEntities.Jobs
.Where(minedJob => identifiedContacts.ContainsKey(minedJob.ContactId));
if (minedJobEntities.IsNotEmpty())
{
HashSet <Guid> contactIds = new HashSet <Guid>(identifiedContacts.Values);
Dictionary <Guid, string> jobs = LoadInfo(contactIds, "Contact", "JobTitle");
foreach (JobEntity minedJob in minedJobEntities)
{
Guid identifiedContact = identifiedContacts[minedJob.ContactId];
string knownJob = jobs[identifiedContact];
if (knownJob == minedJob.Value)
{
duplicateJobs.Add(minedJob.Id);
}
}
}
return(duplicateJobs);
}
private IEnumerable <Guid> FindDuplicateAddresses(Dictionary <Guid, Guid> identifiedContacts,
TextEntities textEntities)
{
List <Guid> duplicateAddresses = new List <Guid>();
IEnumerable <AddressEntity> minedAddressEntities = textEntities.Addresses
.Where(minedAddress => identifiedContacts.ContainsKey(minedAddress.ContactId));
if (minedAddressEntities.IsNotEmpty())
{
HashSet <Guid> contactIds = new HashSet <Guid>(identifiedContacts.Values);
Dictionary <Guid, string> addresses = LoadInfo(contactIds, "Contact", "Address");
foreach (AddressEntity minedAddress in minedAddressEntities)
{
Guid identifiedContact = identifiedContacts[minedAddress.ContactId];
string knownAddress = addresses[identifiedContact];
if (knownAddress == minedAddress.Value)
{
duplicateAddresses.Add(minedAddress.Id);
}
}
}
return(duplicateAddresses);
}
private void AppendSenderEmail(Guid enrchEmailDataId, string senderEmail, bool isFreeDomain,
Guid primaryContact, TextEntities textEntities)
{
if (primaryContact.IsEmpty())
{
return;
}
CommunicationEntity senderEmailEte = new CommunicationEntity {
Id = Guid.NewGuid(),
Value = senderEmail,
ContactId = primaryContact,
Type = EnrchCommunicationType.Email.ToString(),
IsFoundInLastEmail = true,
IsPhone = false,
IsFreeDomain = isFreeDomain
};
if (textEntities.Communications.Contains(senderEmailEte))
{
return;
}
string entityHash = senderEmailEte.GetHash();
var select = (Select) new Select(_userConnection)
.Column(Func.Count("Id"))
.From("EnrchTextEntity")
.Where("Hash").IsEqual(Column.Parameter(entityHash))
.And("ParentId").IsEqual(Column.Parameter(primaryContact));
int emailsWithSameHash = select.ExecuteScalar <int>();
if (emailsWithSameHash > 0)
{
return;
}
textEntities.Communications.Add(senderEmailEte);
CreateTextEntity(enrchEmailDataId, senderEmailEte, EnrchTextEntityType.Communication, primaryContact,
EnrchTextEntitySource.AppDbActivity, entityHash);
}
private Dictionary <Guid, Guid> UpdateContactIdentificationInfo(Guid enrchEmailDataId,
TextEntities textEntities, string senderEmail, bool isNewEnrichedData)
{
HashSet <ContactEntity> contacts = textEntities.Contacts;
Dictionary <Guid, Guid> identifiedContacts = new Dictionary <Guid, Guid>();
foreach (ContactEntity contactEntity in contacts)
{
bool isContactIdentified = false;
if (!isNewEnrichedData)
{
string contactHash = contactEntity.GetHash();
isContactIdentified = CheckActualContactIdentification(contactHash, enrchEmailDataId);
}
if (isContactIdentified)
{
continue;
}
IEnumerable <Guid> contactIds;
ContactIdentificationType identificationType = IdentifyContact(contactEntity, textEntities, senderEmail,
out contactIds);
if (!contactIds.IsNullOrEmpty())
{
if (contactIds.Count() == 1)
{
identifiedContacts.Add(contactEntity.Id, contactIds.First());
}
Guid entityId = GetTextEntityId(enrchEmailDataId, isNewEnrichedData, contactEntity);
if (entityId.IsEmpty())
{
continue;
}
_enrichEntitySearchHelper.AddFoundContacts(entityId, contactIds, identificationType);
}
}
return(identifiedContacts);
}
private void CreateEnrichedTextEntities(Guid enrchEmailDataId, TextEntities emailEntities)
{
foreach (OrganizationEntity organization in emailEntities.Organizations)
{
CreateTextEntity(enrchEmailDataId, organization, EnrchTextEntityType.Organization, Guid.Empty);
}
foreach (ContactEntity contact in emailEntities.Contacts)
{
CreateTextEntity(enrchEmailDataId, contact, EnrchTextEntityType.Contact, contact.OrganizationId);
}
foreach (CommunicationEntity communication in emailEntities.Communications)
{
CreateTextEntity(enrchEmailDataId, communication, EnrchTextEntityType.Communication,
communication.ContactId);
}
foreach (AddressEntity address in emailEntities.Addresses)
{
CreateTextEntity(enrchEmailDataId, address, EnrchTextEntityType.Address, address.ContactId);
}
foreach (JobEntity job in emailEntities.Jobs)
{
CreateTextEntity(enrchEmailDataId, job, EnrchTextEntityType.Job, job.ContactId);
}
}
private ContactIdentificationType IdentifyContact(ContactEntity contactEntity, TextEntities textEntities,
string senderEmail, out IEnumerable <Guid> contactIds)
{
if (contactEntity.IsFoundInLastEmail)
{
contactIds = _contactSearcher.SearchByEmails(new List <string> {
senderEmail
});
if (!contactIds.IsNullOrEmpty())
{
return(ContactIdentificationType.SenderEmail);
}
}
List <string> emails = EnrichEntitySearchHelper.GetEmails(textEntities, contactEntity.Id);
contactIds = _contactSearcher.SearchByEmails(emails);
if (!contactIds.IsNullOrEmpty())
{
return(ContactIdentificationType.Email);
}
contactIds = _contactSearcher.SearchByName(contactEntity.Value);
if (!contactIds.IsNullOrEmpty())
{
return(ContactIdentificationType.Name);
}
contactIds = _contactSearcher.SearchByFirstAndLastName(contactEntity.GivenName, contactEntity.Surname);
if (!contactIds.IsNullOrEmpty())
{
return(ContactIdentificationType.FirstLastName);
}
List <string> phones = EnrichEntitySearchHelper.GetPhones(textEntities, contactEntity.Id);
contactIds = _contactSearcher.SearchByPhones(phones);
if (!contactIds.IsNullOrEmpty())
{
return(ContactIdentificationType.Phone);
}
return(null);
}
/// <summary>
/// Marks mined entities, that are already present in contact, as duplicates.
/// </summary>
/// <param name="identifiedContact">
/// Dictionary, where key is mined contact text entity id and value is identified contact.
/// </param>
/// <param name="entities">Mined entities.</param>
public void DeduplicateContactData(Dictionary <Guid, Guid> identifiedContacts, TextEntities entities)
{
var duplicateCommunications = FindDuplicateCommunications(identifiedContacts, entities);
var duplicateAddresses = FindDuplicateAddresses(identifiedContacts, entities);
var duplicateJobs = FindDuplicateJobs(identifiedContacts, entities);
var duplicates = duplicateCommunications.Concat(duplicateAddresses).Concat(duplicateJobs);
if (duplicates.IsNotEmpty())
{
UpdateDuplicateEntities(duplicates);
}
}
private List <MiningResult> ProcessEntities(Dictionary <Entity, GetEmailEntitiesResponse> minedInfo)
{
var results = new List <MiningResult>();
foreach (KeyValuePair <Entity, GetEmailEntitiesResponse> pairs in minedInfo)
{
Entity activity = pairs.Key;
GetEmailEntitiesResponse response = pairs.Value;
try {
if (response.IsFailure)
{
SetAcivityIsEnriched(activity.PrimaryColumnValue, Guid.Empty, EnrichStatus.Error);
continue;
}
TextEntities emailEntities = response.TextEntities;
string entitiesHash = emailEntities.GetHash();
Entity enrichedRecord = FindEnrichedData(entitiesHash);
bool isNewEnrichedData = enrichedRecord == null;
Guid enrchEmailDataId;
EnrichEmailDataStatus status;
if (isNewEnrichedData)
{
enrichedRecord = CreateNewEnrichedData(emailEntities);
enrchEmailDataId = enrichedRecord.PrimaryColumnValue;
status = EnrichEmailDataStatus.Mined;
CreateEnrichedTextEntities(enrchEmailDataId, emailEntities);
}
else
{
enrchEmailDataId = enrichedRecord.PrimaryColumnValue;
status = enrichedRecord.GetTypedColumnValue <string>("Status");
}
Guid primaryContactTextEntityId =
FindPrimaryContact(emailEntities, enrchEmailDataId, isNewEnrichedData);
string sender = activity.GetTypedColumnValue <string>("Sender");
string senderEmail = sender.ExtractEmailAddress();
Dictionary <Guid, Guid> identifiedContacts = UpdateContactIdentificationInfo(enrchEmailDataId,
emailEntities, senderEmail, isNewEnrichedData);
UpdateAccountIdentificationInfo(enrchEmailDataId, emailEntities, isNewEnrichedData);
if (senderEmail.IsNotNullOrEmpty())
{
AppendSenderEmail(enrchEmailDataId, senderEmail, response.IsSenderEmailDomainFree,
primaryContactTextEntityId, emailEntities);
}
_deduplicator.DeduplicateContactData(identifiedContacts, emailEntities);
results.Add(new MiningResult {
ActivityId = activity.PrimaryColumnValue,
EnrchEmailDataId = enrchEmailDataId,
EnrichStatus = EnrichStatus.Done,
EnrchEmailDataStatus = status
});
SetAcivityIsEnriched(activity.PrimaryColumnValue, enrichedRecord.PrimaryColumnValue,
EnrichStatus.Done);
PublishEmailEnrichmentMetrics(activity);
} catch (Exception ex) {
_log.ErrorFormat("Cannot save enriched data for email id = {0}. Error: {1}",
activity.PrimaryColumnValue, ex);
}
}
return(results);
}
