protected override void DoAction()
{
var entities = GetEntitiesToProcess();
foreach (var entity in entities) {
var queriesForDomain = new HashSet<string>();
entity.Status |= DomainStatus.PhrasesCollected;
try {
for (var page = 1;; page++) {
var hasUnique = false;
var content = SpywordsQueryWrapper.GetQueriesForDomain(entity.Domain, page);
var listLinksToInsert = new List<Domainphrase>();
foreach (Match wordMatch in _siteSpywordsExpractor.Matches(content)) {
var word = wordMatch.Groups["word"].Value.ToLower();
if (queriesForDomain.Contains(word)) {
continue;
}
queriesForDomain.Add(word);
hasUnique = true;
var phrase = Phrase.DataSource
.WhereEquals(Phrase.Fields.Text, word)
.First(Phrase.Fields.ID);
if(phrase == null) {
phrase = new Phrase {
Datecreated = DateTime.UtcNow,
Status = PhraseStatus.NotCollected,
CollectionIdentity = CollectionIdentity,
Text = word
};
phrase.Save();
}
var firstDomainPhrase = Domainphrase.DataSource
.WhereEquals(Domainphrase.Fields.DomainID, entity.ID)
.WhereEquals(Domainphrase.Fields.PhraseID, phrase.ID)
.First();
if (firstDomainPhrase == null) {
var domainphrase = new Domainphrase {
DomainID = entity.ID,
PhraseID = phrase.ID,
SourceType = SourceType.Context,
CollectionIdentity = CollectionIdentity
};
listLinksToInsert.Add(domainphrase);
} else {
firstDomainPhrase.SourceType |= SourceType.Context;
firstDomainPhrase.Save();
}
}
TaskRunner.Instance.AddAction(() => {
listLinksToInsert.Save<Domainphrase, int>();
});
if (!hasUnique) {
break;
}
}
}
catch (Exception ex) {
Logger.Error(ex);
entity.Status |= DomainStatus.PhrasesCollectedError;
}
entity.Save();
}
}
protected static void CreateOrUpdateDomainPhrase(DomainEntity domainEntity, Phrase phrase, SearchEngine seType, SourceType sourceType)
{
var firstDomainPhrase = Domainphrase.DataSource
.WhereEquals(Domainphrase.Fields.DomainID, domainEntity.ID)
.WhereEquals(Domainphrase.Fields.PhraseID, phrase.ID)
.WhereEquals(Domainphrase.Fields.CollectionIdentity, (short) CollectionIdentity)
.First();
if (firstDomainPhrase == null) {
var domainphrase = new Domainphrase {
DomainID = domainEntity.ID,
PhraseID = phrase.ID,
SourceType = sourceType,
SE = seType,
CollectionIdentity = CollectionIdentity
};
domainphrase.Save();
} else {
firstDomainPhrase.SourceType |= sourceType;
firstDomainPhrase.SE |= seType;
firstDomainPhrase.Save();
}
}