private async Task ExtractProps( BookItem Inst, string Content ) { foreach( PropExt Extr in PropDefs ) { if ( !Extr.Enabled ) continue; string PropValue = MatchSingle( Extr, Content ); if ( Extr.SubProc.HasProcedures ) { ProcManager.PanelMessage( this, () => Res.RSTR( "SubProcRun" ), LogType.INFO ); ProcPassThru PPass = new ProcPassThru( new ProcConvoy( this, Inst ) ); ProcConvoy SubConvoy = await Extr.SubProc.CreateSpider().Crawl( new ProcConvoy( PPass, PropValue ) ); // Process ReceivedConvoy if ( SubConvoy.Payload is string ) PropValue = ( string ) SubConvoy.Payload; else if ( SubConvoy.Payload is IEnumerable<string> ) PropValue = string.Join( "\n", ( IEnumerable<string> ) SubConvoy.Payload ); else if ( SubConvoy.Payload is IStorageFile ) PropValue = await ( ( IStorageFile ) SubConvoy.Payload ).ReadString(); else if ( SubConvoy.Payload is IEnumerable<IStorageFile> ) PropValue = await ( ( IEnumerable<IStorageFile> ) SubConvoy.Payload ).First().ReadString(); else continue; } // If the website split a single property into serveral pages // That website is stupid. Would not support. if( !Inst.ReadParam( Extr.PType.ToString(), PropValue.ToCTrad() ) ) { ProcManager.PanelMessage( this, () => Res.RSTR( "InvalidParam", Extr.PType ), LogType.WARNING ); } } }
private async Task SearchBooks( List<BookInstruction> ItemList, ProcPassThru PPass, ProcConvoy KnownBook, string Content ) { ProcFind.RegItem RegParam = new ProcFind.RegItem( ItemPattern, ItemParam, true ); if ( !RegParam.Validate() ) return; MatchCollection matches = RegParam.RegExObj.Matches( Content ); foreach ( Match match in matches ) { if ( HasSubProcs && RegParam.Valid ) { string FParam = string.Format( RegParam.Format , match.Groups .Cast<Group>() .Select( g => g.Value ) .ToArray() ); ProcConvoy ItemConvoy = await ItemProcs.CreateSpider().Crawl( new ProcConvoy( PPass, FParam ) ); string Id = await GetId( ItemConvoy ); if ( string.IsNullOrEmpty( Id ) ) { ProcManager.PanelMessage( this, () => { StringResources stx = new StringResources( "Error" ); return stx.Str( "NoIdForBook" ); }, LogType.WARNING ); continue; } ItemConvoy = ProcManager.TracePackage( ItemConvoy, ( P, C ) => C.Payload is BookInstruction ); if ( !( ItemConvoy == null || ItemConvoy == KnownBook ) ) { BookInstruction BInst = ( BookInstruction ) ItemConvoy.Payload; ItemList.Add( BInst ); if ( HasBookSpider ) BInst.PlaceDefs( Id, BookSpider ); } else { ProcManager.PanelMessage( this, () => { StringResources stx = new StringResources( "Error" ); return stx.Str( "NotABook" ); }, LogType.WARNING ); } } } }
public override async Task<ProcConvoy> Run( ProcConvoy Convoy ) { Convoy = await base.Run( Convoy ); ProcConvoy UsableConvoy; if ( !TryGetConvoy( out UsableConvoy, ( P, C ) => { if ( P.Type.HasFlag( ProcType.FIND ) || P.Type.HasFlag( ProcType.URLLIST ) ) { return C.Payload is IEnumerable<IStorageFile>; } return false; } ) ) return Convoy; IInstructionSet SpTOC = null; // Search for the closest Instruction Set ProcConvoy SpiderInst = ProcManager.TracePackage( Convoy , ( P, C ) => { return C.Payload is IInstructionSet; } ); if( SpiderInst != null ) { SpTOC = SpiderInst.Payload as IInstructionSet; } if( SpTOC == null ) { SpTOC = new BookInstruction(); } IEnumerable<IStorageFile> ISFs = UsableConvoy.Payload as IEnumerable<IStorageFile>; bool VTitleAddOnce = false; ProcPassThru PPass = new ProcPassThru( new ProcConvoy( this, SpTOC ) ); foreach( IStorageFile ISF in ISFs ) { string Content = await ISF.ReadString(); ProcFind.RegItem RegTitle = new ProcFind.RegItem( VolPattern, VolTitle, true ); ProcFind.RegItem RegParam = new ProcFind.RegItem( VolPattern, VolParam, true ); if ( !( RegTitle.Validate() || RegParam.Validate() ) ) continue; MatchCollection matches = RegTitle.RegExObj.Matches( Content ); foreach ( Match match in matches ) { if ( VTitleAddOnce ) break; VolInstruction VInst = null; if( RegTitle.Valid ) { string FTitle = string.Format( RegTitle.Format , match.Groups .Cast<Group>() .Select( g => g.Value ) .ToArray() ); if( string.IsNullOrEmpty( RegTitle.Pattern ) ) { VTitleAddOnce = true; } VInst = new VolInstruction( VTitleAddOnce ? SpTOC.LastIndex : match.Index , FTitle.ToCTrad() ); VInst.ProcMan = VolProcs; SpTOC.PushInstruction( VInst ); } else { ProcManager.PanelMessage( this, () => Res.RSTR( "InvalidPattern" ), LogType.WARNING ); continue; } if ( HasVolProcs && RegParam.Valid ) { string FParam = string.Format( RegParam.Format , match.Groups .Cast<Group>() .Select( g => g.Value ) .ToArray() ); if( VolAsync ) { VInst.SetProcId( VolProcs.GUID ); VInst.PushConvoyParam( FParam ); } else { ProcConvoy VolConvoy = await VolProcs.CreateSpider().Crawl( new ProcConvoy( PPass, VInst ) ); } } } RegTitle = new ProcFind.RegItem( EpPattern, EpTitle, true ); RegParam = new ProcFind.RegItem( EpPattern, EpParam, true ); if ( !( RegTitle.Validate() || RegParam.Validate() ) ) continue; matches = RegTitle.RegExObj.Matches( Content ); foreach ( Match match in matches ) { EpInstruction EInst = null; if( RegTitle.Valid ) { string FTitle = string.Format( RegTitle.Format , match.Groups .Cast<Group>() .Select( g => g.Value ) .ToArray() ); EInst = new EpInstruction( VTitleAddOnce ? SpTOC.LastIndex : match.Index , FTitle.ToCTrad() ); EInst.ProcMan = EpProcs; SpTOC.PushInstruction( EInst ); } else { ProcManager.PanelMessage( this, () => Res.RSTR( "InvalidPattern" ), LogType.WARNING ); continue; } if ( HasEpProcs && RegParam.Valid ) { string FParam = string.Format( RegParam.Format , match.Groups .Cast<Group>() .Select( g => g.Value ) .ToArray() ); if( EpAsync ) { EInst.SetProcId( EpProcs.GUID ); EInst.PushConvoyParam( FParam ); } else { ProcConvoy EpConvoy = await EpProcs.CreateSpider().Crawl( new ProcConvoy( PPass, FParam ) ); } } } } return new ProcConvoy( this, SpTOC ); }
public override async Task<ProcConvoy> Run( ProcConvoy Convoy ) { Convoy = await base.Run( Convoy ); ProcConvoy UsableConvoy; if ( !TryGetConvoy( out UsableConvoy , ( P, C ) => C.Payload is IEnumerable<IStorageFile> || C.Payload is IEnumerable<string> || C.Payload is IStorageFile || C.Payload is string ) ) return Convoy; List<BookInstruction> SpItemList = null; // Search for the closest Instruction Set ProcConvoy SpiderInst = ProcManager.TracePackage( Convoy , ( P, C ) => C.Payload is IEnumerable<BookInstruction> ); if ( SpiderInst != null ) { SpItemList = ( List<BookInstruction> ) SpiderInst.Payload; } if ( SpItemList == null ) { SpItemList = new List<BookInstruction>(); } ProcPassThru PPass = new ProcPassThru( new ProcConvoy( this, SpItemList ) ); ProcConvoy KnownBook = ProcManager.TracePackage( Convoy, ( P, C ) => C.Payload is BookInstruction ); if ( UsableConvoy.Payload is IEnumerable<IStorageFile> ) { IEnumerable<IStorageFile> ISFs = ( IEnumerable<IStorageFile> ) UsableConvoy.Payload; foreach ( IStorageFile ISF in ISFs ) { string Content = await ISF.ReadString(); await SearchBooks( SpItemList, PPass, KnownBook, Content ); } } else if ( UsableConvoy.Payload is IEnumerable<string> ) { IEnumerable<string> Contents = ( IEnumerable<string> ) UsableConvoy.Payload; foreach ( string Content in Contents ) { await SearchBooks( SpItemList, PPass, KnownBook, Content ); } } else if ( UsableConvoy.Payload is IStorageFile ) { IStorageFile ISF = ( IStorageFile ) UsableConvoy.Payload; string Content = await ISF.ReadString(); await SearchBooks( SpItemList, PPass, KnownBook, Content ); } else // string { await SearchBooks( SpItemList, PPass, KnownBook, ( string ) UsableConvoy.Payload ); } return new ProcConvoy( this, SpItemList ); }