public override ChunkContentCandidateCollection GetCandidates(ChunkDetectionResult results) { JunctionGraphMetrics <NodeGraph> junctionGraphMetrics = new JunctionGraphMetrics <NodeGraph>(); junctionGraphMetrics.Process(results.CurrentGraph); var junctionBlocks = junctionGraphMetrics.GetJunctionBlocks(1, 4, true, true); List <JunctionPoint <NodeGraph> > sorted = junctionBlocks.OrderByDescending(x => x.JunctionSize).ToList(); sorted = sorted.OrderByDescending(x => x.XPathRoot.getPathParts().Count).ToList(); ChunkContentCandidateCollection output = new ChunkContentCandidateCollection(); foreach (var junctionPoint in sorted) { NodeGraph childNode = results.CurrentGraph.GetChildAtPath(junctionPoint.XPathRoot, "/", false); ChunkContentCandidate newCandidate = new ChunkContentCandidate(this, childNode); newCandidate.MetaData = junctionPoint; output.Add(newCandidate); } results.Candidates.AddRange(output); return(output); }
public override Boolean SetContentChunk(ContentChunk chunk, ChunkContentCandidate candidate, ChunkDetectionResult result) { chunk.multiNodePolicy = TaskMultiNodePolicy.AsSeparatedTables; chunk.type = ContentChunkType.DynamicDataExtraction; return(true); }
public override ChunkContentCandidateCollection GetCandidates(ChunkDetectionResult results) { ChunkContentCandidateCollection output = new ChunkContentCandidateCollection(); var tableChildren = results.CurrentGraph.getAllChildren(HtmlPathSelectExpression); foreach (NodeGraph tableChild in tableChildren) { ChunkContentCandidate newCandidate = new ChunkContentCandidate(this, tableChild); output.Add(newCandidate); } results.Candidates.AddRange(output); return(output); }
public override bool SetContentChunk(ContentChunk chunk, ChunkContentCandidate candidate, ChunkDetectionResult result) { JunctionPoint <NodeGraph> junctionPoint = candidate.MetaData as JunctionPoint <NodeGraph>; NodeGraphTemplateDetection templateDetection = new NodeGraphTemplateDetection(junctionPoint.rootItem); if (templateDetection.IsValid()) { RecordTemplateSet templateSet = templateDetection.GetTemplateSet(); chunk.ExtractorCustomizationSettings.AddObjectEntry(nameof(RecordTemplateExtractor.TemplateSet), templateSet); return(true); } else { return(false); } }
public static Int32 Compare(ChunkContentCandidate CandidateA, ChunkContentCandidate CandidateB) { if (CandidateA.ScoreModel == null && CandidateB.ScoreModel == null) { return(0); } if (CandidateB.ScoreModel == null) { return(1); } if (CandidateA.ScoreModel == null) { return(-1); } var pathPartsA = CandidateA.Node.path.SplitSmart("/"); var pathPartsB = CandidateB.Node.path.SplitSmart("/"); if (pathPartsA.Count == pathPartsB.Count) { var dynamicScoreComparison = CandidateA.ScoreModel.DynamicNodeScore.CompareTo(CandidateB.ScoreModel.DynamicNodeScore); if (dynamicScoreComparison == 0) { return(CandidateA.ScoreModel.TemplateInstances.CompareTo(CandidateB.ScoreModel.TemplateInstances)); } else { return(dynamicScoreComparison); } } else { if (pathPartsA.Count > pathPartsB.Count) { return(1); } else { return(-1); } } }
public override bool SetContentChunk(ContentChunk chunk, ChunkContentCandidate candidate, ChunkDetectionResult result) { NodeGraphTemplateDetection templateDetection = new NodeGraphTemplateDetection(candidate.Node); chunk.multiNodePolicy = TaskMultiNodePolicy.AsSeparatedTables; chunk.type = ContentChunkType.DynamicDataExtraction; if (templateDetection.IsValid()) { RecordTemplateSet templateSet = templateDetection.GetTemplateSet(); chunk.ExtractorCustomizationSettings.AddObjectEntry(nameof(RecordTemplateExtractor.TemplateSet), templateSet); return(true); } else { chunk.description = "Refused for failed template detection"; return(false); } }
public override ChunkContentCandidateCollection GetCandidates(ChunkDetectionResult results) { ChunkContentCandidateCollection output = new ChunkContentCandidateCollection(); var inputGraph = results.InitialGraph; //.CloneByItems(); List <NodeGraph> peaks = new List <NodeGraph>(); ListDictionary <String, NodeGraph> dynamicNodes = new ListDictionary <string, NodeGraph>(); var dynamicNodeList = inputGraph.GetChildrenWithItemSet().Where(x => x.item.Category.HasFlag(NodeInTemplateRole.Dynamic)).ToList(); NodeGraph dynamicNode = dynamicNodeList.FirstOrDefault(); Int32 i = 0; Int32 i_limit = 5000; while (dynamicNode != null) { var nodePeakSearch = new JunctionPeakSearch(dynamicNode); NodeGraph peakNode = nodePeakSearch.GetJunctionPeak(Convert.ToDouble(MinJunctionSize)); dynamicNodes[dynamicNode.path].Add(dynamicNode); if (dynamicNodes[dynamicNode.path].Count > 1) { } if (peakNode == null) { //dynamicNode.removeFromParent(); } else { if (peaks.Contains(peakNode)) { } else { peaks.Add(peakNode); if (peakNode.level < MinPeakNodeLevel) { // dynamicNode.removeFromParent(); } else { //var peakNodeAtSource = results.InitialGraph.GetChildAtPath(peakNode.path, "/", false); if (peakNode.Count() != 0) { ChunkContentCandidate newCandidate = new ChunkContentCandidate(this, peakNode); // peakNode.removeFromParent(); output.Add(newCandidate); } else { } } } } i++; dynamicNode = dynamicNodeList.FirstOrDefault(x => dynamicNodes[x.path].Count == 0); // inputGraph.GetChildrenWithItemSet().FirstOrDefault(x => (x.item.Category.HasFlag(NodeInTemplateRole.Dynamic) && dynamicNodes[x.path].Count == 0)); if (i > i_limit) { break; } } if (peaks.Count > 0) { } results.Candidates.AddRange(output); return(output); }
public abstract Boolean SetContentChunk(ContentChunk chunk, ChunkContentCandidate candidate, ChunkDetectionResult result);