public void AddIndexingData(string domainUrl, IndexingData data)
{
var filter = Builders <Domain> .Filter.Where(x => x.Url == domainUrl);
var update = Builders <Domain> .Update.Push("IndexingStats", data);
Collection.FindOneAndUpdate(filter, update);
}
public async Task AddIndexingDataAsync(string domainUrl, IndexingData data)
{
var filter = Builders <Domain> .Filter.Where(x => x.Url == domainUrl);
var update = Builders <Domain> .Update.Push("IndexingStats", data);
await Collection.FindOneAndUpdateAsync(filter, update);
}
public void UpdateCurrentStats(string domainUrl, IndexingData data)
{
var filter = Builders <Domain> .Filter;
var domainUrlAndStatsDateFilter = filter.And(
filter.Eq(x => x.Url, domainUrl),
filter.ElemMatch(x => x.IndexingStats,
c => c.ProcessingDate.Equals(data.ProcessingDate.Date)));
var domain = Collection.Find(domainUrlAndStatsDateFilter).SingleOrDefault();
// update with positional operator
var update = Builders <Domain> .Update;
var indexingStatsSetter = update.Set("IndexingStats.$.PagesNumber", data.PagesNumber);
Collection.UpdateOne(domainUrlAndStatsDateFilter, indexingStatsSetter);
}
private void ReIndexLeadScoreContact(int contactId, int accountId, int score)
{
var indexingData = new IndexingData()
{
EntityIDs = new List <int>()
{
contactId
},
IndexType = (int)IndexType.Contacts
};
accountService.InsertIndexingData(new Messaging.Accounts.InsertIndexingDataRequest()
{
IndexingData = indexingData
});
addToTopic(contactId, accountId, score);
}
public void UpdateCurrentStats(string domainUrl, IndexingData data)
{
_uow.DomainRepository.UpdateCurrentStats(domainUrl, data);
}
public void AddIndexingData(string domainUrl, IndexingData data)
{
_uow.DomainRepository.AddIndexingData(domainUrl, data);
}
public async Task AddIndexingDataAsync(string domainUrl, IndexingData data)
{
await _uow.DomainRepository.AddIndexingDataAsync(domainUrl, data);
}
private void HandleDomainUrl(ParserInput message)
{
IndexingData indexingData = null;
using (var webClient = new WebClient())
{
if (message.UseProxy)
{
webClient.Proxy = _proxyProvider.GetProxy();
}
var url = _proxyProvider.GetRequestUrl(string.Format(GoogleSearchUrl, message.Url));
webClient.Headers.Add("Accept-Language", "en-US");
var pageData = webClient.DownloadString(url);
var page = new HtmlDocument();
page.LoadHtml(pageData);
HtmlNode resultStatsNode = page.GetElementbyId("resultStats");
if (resultStatsNode == null || resultStatsNode.InnerHtml == string.Empty)
{
indexingData = new IndexingData
{
PagesNumber = 0,
ProcessingDate = DateTime.Now.Date
};
var domainStat = new DomainStat {
DomainURL = message.Url, IndexingData = indexingData
};
Sender.Tell(domainStat, Self);
return;
}
var resultStats = HtmlEntity.DeEntitize(resultStatsNode.InnerHtml);
var firstNumberIndex = resultStats.IndexOfAny("123456789".ToCharArray());
var lastNumberIndex = resultStats.LastIndexOfAny("0123456789".ToCharArray());
if (firstNumberIndex >= 0 && lastNumberIndex >= 0 && lastNumberIndex >= firstNumberIndex)
{
var match = resultStats.Substring(firstNumberIndex, lastNumberIndex - firstNumberIndex + 1);
var numberString = match.Replace(",", string.Empty).Replace(".", string.Empty);
numberString = Regex.Replace(numberString, @"\s+", "");
var indexedPagesNumber = long.Parse(numberString);
indexingData = new IndexingData
{
PagesNumber = indexedPagesNumber,
ProcessingDate = DateTime.Now.Date
};
var domainStat = new DomainStat {
DomainURL = message.Url, IndexingData = indexingData
};
Sender.Tell(domainStat, Self);
}
else
{
var errorMessage = "Match was not successful! Result stats: " + resultStats;
var parsingError = new ParsingError {
DomainURL = message.Url, ErrorMessage = errorMessage
};
Sender.Tell(parsingError, Self);
}
}
}
public async Task ProcessDataAsync(CancellationToken cancellationToken = default)
{
var indexingData = new IndexingData();
stopwatch.Start();
this.logger.LogInformation("Fetching and parsing xml users.");
var users = await this.linqRepository.UsersAsync(cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("sending users to store.");
await this.searchService.AddUsersAsync(users, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
stopwatch.Start();
this.logger.LogInformation("Fetching and parsing xml posts.");
var posts = await this.linqRepository.PostsAsync(cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("sending posts to store.");
await this.searchService.AddPostsAsync(posts, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
stopwatch.Start();
this.logger.LogInformation("Fetching and parsing xml comments.");
var comments = await this.linqRepository.CommentsAsync(cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("sending comments to store.");
await this.searchService.AddCommentsAsync(comments, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
stopwatch.Start();
this.logger.LogInformation("Fetching and parsing xml tags.");
var tags = await this.linqRepository.TagsAsync(cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("sending tags to store.");
await this.searchService.AddTagsAsync(tags, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
stopwatch.Start();
this.logger.LogInformation("Fetching and parsing xml votes.");
var votes = await this.linqRepository.VotesAsync(cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("sending votes to store.");
await this.searchService.AddVotesAsync(votes, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
stopwatch.Start();
this.logger.LogInformation("Fetching and parsing xml badges.");
var badges = await this.linqRepository.BadgesAsync(cancellationToken).ConfigureAwait(false);
this.logger.LogInformation("sending badges to store.");
await this.searchService.AddBadgesAsync(badges, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
elapsed += stopwatch.ElapsedMilliseconds;
this.logger.LogInformation("Sending indexing data to store.");
indexingData.CreationDate = DateTime.UtcNow;
indexingData.DocumentThresholdAdjustment = SearchService.ChunkIterationSize;
indexingData.ErrorBreakpoint = this.searchService.ErrorCountBreakPoint;
indexingData.MaximumChunkSize = SearchService.MaximumChunkSize;
indexingData.RetryFailedChunks = this.searchService.RetryFailedChunks;
indexingData.RunningTime = stopwatch.ElapsedMilliseconds;
indexingData.TotalDocumentsIndexed =
(badges.Count + votes.Count + tags.Count + comments.Count + posts.Count + users.Count) -
this.searchService.FinalFailureCount;
indexingData.TotalErrors = this.searchService.TotalFailures;
indexingData.FinalFailureCount = this.searchService.FinalFailureCount;
}
protected override void ExecuteInternal(IJobExecutionContext context)
{
var messages = _messageService
.GetLeadScoreMessages()
.Messages
.ToArray();
if (!messages.IsAny())
{
return;
}
var auditedMessages = new List <LeadScoreMessage>();
Log.Informational($"Received messages count: {messages.Length}");
foreach (var message in messages)
{
try
{
Log.Informational($"Processing LeadScore Message message {message}");
GetLeadScoreRuleByConditionResponse leadScoreRuleResponse;
string conditionValue;
int entityId;
SetCondtionValueAndEntityValue(message, out entityId, out conditionValue);
var leadScoreRuleRequest = new GetLeadScoreRuleByConditionRequest
{
AccountId = message.AccountID,
Condition = (LeadScoreConditionType)message.LeadScoreConditionType,
ConditionValue = conditionValue,
EntityID = entityId
};
if (message.LeadScoreConditionType == (byte)LeadScoreConditionType.ContactClicksLink)
{
leadScoreRuleResponse = _leadScoreRuleService.GetCampaignClickLeadScoreRule(leadScoreRuleRequest);
}
else
{
leadScoreRuleResponse = _leadScoreRuleService.GetLeadScoreRules(leadScoreRuleRequest);
}
Log.Informational($"Audit leadscore, condtion value: {conditionValue}, entity id:{entityId}, message {message.ToString()}");
if (leadScoreRuleResponse.Rules != null && leadScoreRuleResponse.Rules.Any())
{
Log.Informational(string.Format("Audit leadscore count, Rule ID: {0}", leadScoreRuleResponse.Rules.Count(), leadScoreRuleResponse.Rules.Select(s => s.Id)));
var response = _leadScoreService.IsScoreAudited(
new LeadScoreAuditCheckRequest
{
AccountId = message.AccountID,
ContactId = message.ContactID,
Condition = (LeadScoreConditionType)message.LeadScoreConditionType,
ConditionValue = conditionValue,
EntityId = entityId,
Rules = leadScoreRuleResponse.Rules
});
if (!response.IsAudited && (response.UnAuditedRules != null && response.UnAuditedRules.Any()))
{
Log.Informational($"Update leadscore, message {message.ToString()}");
Log.Informational(string.Format("while inserting Audit leadscore count, Rule ID: {0}", leadScoreRuleResponse.Rules.Count(), leadScoreRuleResponse.Rules.Select(s => s.Id)));
_leadScoreService.InsertLeadScore(new InsertLeadScoreRequest()
{
Condition = (LeadScoreConditionType)message.LeadScoreConditionType,
ConditionValue = conditionValue,
ContactId = message.ContactID,
AccountId = message.AccountID,
RequestedBy = message.UserID,
EntityId = entityId,
Rules = response.UnAuditedRules
});
message.Remarks = "Lead score applied";
message.LeadScoreProcessStatusID = (int)TrackMessageProcessStatus.Processed;
auditedMessages.Add(message);
}
else if (response.IsAudited)
{
Log.Informational($"Leadscore already applied, message {message.ToString()}");
message.LeadScoreProcessStatusID = (int)TrackMessageProcessStatus.Processed;
message.Remarks = "Lead score is already applied";
}
}
else if (leadScoreRuleResponse.Rules == null || !leadScoreRuleResponse.Rules.Any())
{
Log.Informational($"No rule defined, message {message.ToString()}");
message.Remarks = "No rule defined";
message.LeadScoreProcessStatusID = (int)TrackMessageProcessStatus.Ignored;
}
message.ProcessedOn = DateTime.UtcNow;
}
catch (Exception ex)
{
Log.Informational($"Error occurred while processing leadscore {message} {ex}");
message.ProcessedOn = DateTime.MinValue;
message.LeadScoreProcessStatusID = (int)TrackMessageProcessStatus.Error;
message.Remarks = $"Lead score not processed, {ex}";
}
}
_messageService.UpdateLeadScoreMessage(new UpdateLeadScoreMessage()
{
Messages = messages
});
//update all messages
if (auditedMessages.Any())
{
//index all contacts messages
var indexingData = new IndexingData
{
EntityIDs = auditedMessages.Select(m => m.ContactID).ToList(),
IndexType = (int)IndexType.Contacts
};
_accountService.InsertIndexingData(new InsertIndexingDataRequest {
IndexingData = indexingData
});
}
}