public void AddCandidateLink(ContextResult contextResult, string link)
{
if (!VisitedLinks.Contains(link) && !contextResult.CandidatesToRecursion.Contains(link))
contextResult.CandidatesToRecursion.Add(link);
}
private ContextResult Execute(Instruction instruction)
{
//cannot execute if it is not recurrent and it was already run
ContextResult context = new ContextResult();
if (instruction != null)
{
InitializeInstructionExecution(instruction);
try
{
context = ExecuteByType(instruction);
}
catch
{
int i = 0;
//Log events
}
SaveAndIndexResults(context, instruction);
FinalizeInstructionExecution(instruction);
}
return context;
}
private void SaveAndIndexResults(ContextResult context, Instruction instruction)
{
Crawler crawler = instruction.GetEngine();
if (crawler.IsCustomSaved)
crawler.CustomSave(context);
else
{
List<CrawlerResult> toSave = new List<CrawlerResult>();
foreach (var result in context.Results)
{
if (!ContentExists(result.Url))
{
db.CrawlerResults.Add(result);
toSave.Add(result);
}
}
if (toSave.Count > 0)
{
try
{
db.SaveChanges();
}
catch (DbEntityValidationException dbEx)
{
foreach (var validationErrors in dbEx.EntityValidationErrors)
{
foreach (var validationError in validationErrors.ValidationErrors)
{
string p = validationError.PropertyName;
string e = validationError.ErrorMessage;
//Trace.TraceInformation("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage);
}
}
}
Index(toSave);
}
}
if (instruction.IsRecursive)
{
Instruction recursiveInstruction = (Instruction)instruction.Clone();
recursiveInstruction.IsOriginal = false;
foreach (string link in context.CandidatesToRecursion)
{
recursiveInstruction.Url = link;
Execute(recursiveInstruction);
}
}
}
public virtual void CustomSave(ContextResult context)
{
throw new Exception("Not yet implemented");
}
