protected void ExtractAutoDetect(
ExtractionItem extractionItem,
IDictionary <string, ExtractionItem> extractionItems,
CollectionDictionary <string, string> extractedItems,
ResponseParserPositionPointer?relativeLocationBase = null)
{
if (extractedItems.ContainsKey(extractionItem.Name))
{
return;
}
// First, extract item as a normal item
ExtractItem(extractionItem, extractionItems, extractedItems, relativeLocationBase);
// Then, add specific functionality, like link-scoped items and registering a ExtractedLink entity
// If item is a link, extract it's link aspect
ExtractionLink extractionLink;
if ((extractionLink = extractionItem as ExtractionLink) != null)
{
ExtractLink(extractionLink);
}
// If item is a frame (which is a link as well), then it's link data has already been extracted an we only need to perform Frame-specific actions, like download and replace values in ExtractedItems
ExtractionFrame extractionFrame;
if ((extractionFrame = extractionItem as ExtractionFrame) != null)
{
// Frames are stored separated from links, to avoid queuing and download by the crawler
ExtractedFrames[extractionFrame.Name] = ExtractedLinks[extractionFrame.Name];
ExtractedLinks.Remove(extractionFrame.Name);
// TODO: Download frames inline and store them in ExtractedItems (override initially extracted values)
var frameResourceLinks = ExtractedFrames.GetValues(extractionFrame.Name);
var frameDownloadTasks = frameResourceLinks
.Select(frame => CrawlingEngine.CrawlAsync(frame, false))
.ToArray();
Task.WaitAll(frameDownloadTasks); // We're not in async context, so we'll have to hold this thread until we download all the inline downloads required
// Replace previously extracted data for the frame with it's downloaded content
ExtractedItems[extractionFrame.Name] =
frameDownloadTasks
.SelectMany(frameDownloadTask => frameDownloadTask.Result)
.OfType <ResponseStringUnit>()
.Select(frameResponse =>
{
IEnumerable <string> result = new [] { frameResponse.Content };
if (extractionFrame.PostProcessOnDownload)
{
result = PostProcess(result, extractionItem.PostProcessors, DependencyDataSource);
}
return(result.ToArray());
})
.SelectMany(pred => pred)
.ToArray();
}
}
public bool Validate()
{
var result = true;
foreach (var required in Required)
{
if (!Dictionary.ContainsKey(required))
{
Trace.TraceError($"Required item {required} missing from dictionary items [{string.Join(",", Dictionary.Keys)}]");
result = false;
}
}
return(result);
}
public IEnumerable <string> ResolveAll(StringWithDependencies stringWithDependencies)
{
if (stringWithDependencies == null)
{
yield return(null);
}
if (!stringWithDependencies.RequiresResolve)
{
yield return(stringWithDependencies.FormatString);
}
var resolvedValues = new CollectionDictionary <string>();
foreach (var dependencyName in stringWithDependencies.DependencyNames)
{
if (ExtractedItems.ContainsKey(dependencyName))
{
var dependencyValues = ExtractedItems[dependencyName];
foreach (var dependencyValue in dependencyValues)
{
resolvedValues.AddValue(dependencyName, dependencyValue);
}
}
else if (ConfigPredefinedValues.Dictionary.ContainsKey(dependencyName))
{
var dependencyValue = ConfigPredefinedValues.Dictionary[dependencyName];
resolvedValues.AddValues(dependencyName, dependencyValue);
}
else if (JobPredefinedValues != null && JobPredefinedValues.Dictionary.ContainsKey(dependencyName))
{
var dependencyValue = JobPredefinedValues.Dictionary[dependencyName];
resolvedValues.AddValues(dependencyName, dependencyValue);
}
else
{
resolvedValues.AddValue(dependencyName, string.Empty);
Trace.TraceError($"{GetType().Name}.ResolveAll: Could not resolve item {dependencyName} with dependencies [{string.Join(",", stringWithDependencies.DependencyNames)}] based on extracted items {ExtractedItems.Select(pred => string.Format("[{0}: {1}]", pred.Key, string.Join(",", pred.Value)))}");
}
}
foreach (var combination in FormatAllDependencyCombinations(stringWithDependencies, resolvedValues))
{
yield return(combination);
}
}
protected void ExtractItem(
ExtractionItem extractionItem,
IDictionary <string, ExtractionItem> extractionItems,
CollectionDictionary <string, string> extractedItems,
ResponseParserPositionPointer?relativeLocationBase = null
)
{
if (extractedItems.ContainsKey(extractionItem.Name))
{
return; // already extracted as someone's dependency
}
ExtractDependencies(extractionItem, extractionItems, extractedItems, relativeLocationBase);
var extractedValues = new List <string>();
// constant value, if specified in config
if (extractionItem.Value != null)
{
extractedValues.Add(DependencyDataSource.Resolve(extractionItem.Value));
}
// values, extracted from page using selector
if (extractionItem.Location != null)
{
var documentParsers = new List <DocumentProcessor>();
// If Context is not specified for item, then item is extracted from base document, without any complexities
if (extractionItem.Context == null)
{
documentParsers.Add(this);
}
else
{
// If Context is specified, we're creating a separate DocumentParser for every context item
// Create once per Context item, not parse the document 10 times for each element extracted
var contextItemName = DependencyDataSource.Resolve(extractionItem.Context.ContextItemName);
if (!ContextItemDocumentParsers.TryGetValue(contextItemName, out var contextDocumentParsers))
{
// Generate context parsers for this ContextItem if they have not been generated before
ContextItemDocumentParsers[contextItemName]
= contextDocumentParsers
= new List <DocumentProcessor>();
var contextResourceFrames = ExtractedFrames[contextItemName];
var contextDocumentLinks = contextResourceFrames.OfType <DocumentLink>().ToArray();
Debug.Assert(contextResourceFrames.Count == contextDocumentLinks.Length);
var contextDocumentStrings = ExtractedItems.GetValues(contextItemName);
Debug.Assert(contextDocumentLinks.Length == contextDocumentStrings.Count);
for (var i = 0; i < contextDocumentLinks.Length; i++)
{
// TODO: Documents and Downloaded strings order is not the same. Refactor.
var documentString = contextDocumentStrings[i];
var documentLink = contextDocumentLinks[i];
var contextDocumentParser =
_documentLink.Config.CreateDocumentStringParser(
documentString,
documentLink,
extractionItem.Context.ContextDocumentType
);
// Reusing document parsers globally inside the config. Otherwise we get infinite recursive
contextDocumentParser.ContextItemDocumentParsers = ContextItemDocumentParsers;
contextDocumentParser.ExtractedItems = ExtractedItems;
contextDocumentParser.ExtractedLinks = ExtractedLinks;
contextDocumentParser.ExtractedFrames = ExtractedFrames;
contextDocumentParsers.Add(contextDocumentParser);
contextDocumentParser.Parse();
}
}
documentParsers.AddRange(contextDocumentParsers);
}
foreach (var documentParser in documentParsers)
{
extractedValues.AddRange(
documentParser.ExtractItemValuesFromLocation(extractionItem.Location, relativeLocationBase)
.Select(pred => pred.ExtractedValue)
);
}
}
// TODO: Reconsider architecture
// Links are not post-processed at all and frames are post-processed on download only
// apply post-processing, if specified
extractedItems.AddValues(
extractionItem.Name,
extractionItem.PostProcessOnExtraction
? PostProcess(extractedValues, extractionItem.PostProcessors, DependencyDataSource)
: extractedValues
);
}