private async Task ExtractProps( BookItem Inst, string Content )
{
foreach( PropExt Extr in PropDefs )
{
if ( !Extr.Enabled ) continue;
string PropValue = MatchSingle( Extr, Content );
if ( Extr.SubProc.HasProcedures )
{
ProcManager.PanelMessage( this, () => Res.RSTR( "SubProcRun" ), LogType.INFO );
ProcPassThru PPass = new ProcPassThru( new ProcConvoy( this, Inst ) );
ProcConvoy SubConvoy = await Extr.SubProc.CreateSpider().Crawl( new ProcConvoy( PPass, PropValue ) );
// Process ReceivedConvoy
if ( SubConvoy.Payload is string )
PropValue = ( string ) SubConvoy.Payload;
else if ( SubConvoy.Payload is IEnumerable<string> )
PropValue = string.Join( "\n", ( IEnumerable<string> ) SubConvoy.Payload );
else if ( SubConvoy.Payload is IStorageFile )
PropValue = await ( ( IStorageFile ) SubConvoy.Payload ).ReadString();
else if ( SubConvoy.Payload is IEnumerable<IStorageFile> )
PropValue = await ( ( IEnumerable<IStorageFile> ) SubConvoy.Payload ).First().ReadString();
else continue;
}
// If the website split a single property into serveral pages
// That website is stupid. Would not support.
if( !Inst.ReadParam( Extr.PType.ToString(), PropValue.ToCTrad() ) )
{
ProcManager.PanelMessage( this, () => Res.RSTR( "InvalidParam", Extr.PType ), LogType.WARNING );
}
}
}
private async Task SearchBooks( List<BookInstruction> ItemList, ProcPassThru PPass, ProcConvoy KnownBook, string Content )
{
ProcFind.RegItem RegParam = new ProcFind.RegItem( ItemPattern, ItemParam, true );
if ( !RegParam.Validate() ) return;
MatchCollection matches = RegParam.RegExObj.Matches( Content );
foreach ( Match match in matches )
{
if ( HasSubProcs && RegParam.Valid )
{
string FParam = string.Format(
RegParam.Format
, match.Groups
.Cast<Group>()
.Select( g => g.Value )
.ToArray()
);
ProcConvoy ItemConvoy = await ItemProcs.CreateSpider().Crawl( new ProcConvoy( PPass, FParam ) );
string Id = await GetId( ItemConvoy );
if ( string.IsNullOrEmpty( Id ) )
{
ProcManager.PanelMessage( this, () =>
{
StringResources stx = new StringResources( "Error" );
return stx.Str( "NoIdForBook" );
}, LogType.WARNING );
continue;
}
ItemConvoy = ProcManager.TracePackage( ItemConvoy, ( P, C ) => C.Payload is BookInstruction );
if ( !( ItemConvoy == null || ItemConvoy == KnownBook ) )
{
BookInstruction BInst = ( BookInstruction ) ItemConvoy.Payload;
ItemList.Add( BInst );
if ( HasBookSpider ) BInst.PlaceDefs( Id, BookSpider );
}
else
{
ProcManager.PanelMessage( this, () =>
{
StringResources stx = new StringResources( "Error" );
return stx.Str( "NotABook" );
}, LogType.WARNING );
}
}
}
}
public override async Task<ProcConvoy> Run( ProcConvoy Convoy )
{
Convoy = await base.Run( Convoy );
ProcConvoy UsableConvoy;
if ( !TryGetConvoy( out UsableConvoy, ( P, C ) =>
{
if ( P.Type.HasFlag( ProcType.FIND ) || P.Type.HasFlag( ProcType.URLLIST ) )
{
return C.Payload is IEnumerable<IStorageFile>;
}
return false;
}
) ) return Convoy;
IInstructionSet SpTOC = null;
// Search for the closest Instruction Set
ProcConvoy SpiderInst = ProcManager.TracePackage(
Convoy
, ( P, C ) => {
return C.Payload is IInstructionSet;
}
);
if( SpiderInst != null )
{
SpTOC = SpiderInst.Payload as IInstructionSet;
}
if( SpTOC == null )
{
SpTOC = new BookInstruction();
}
IEnumerable<IStorageFile> ISFs = UsableConvoy.Payload as IEnumerable<IStorageFile>;
bool VTitleAddOnce = false;
ProcPassThru PPass = new ProcPassThru( new ProcConvoy( this, SpTOC ) );
foreach( IStorageFile ISF in ISFs )
{
string Content = await ISF.ReadString();
ProcFind.RegItem RegTitle = new ProcFind.RegItem( VolPattern, VolTitle, true );
ProcFind.RegItem RegParam = new ProcFind.RegItem( VolPattern, VolParam, true );
if ( !( RegTitle.Validate() || RegParam.Validate() ) ) continue;
MatchCollection matches = RegTitle.RegExObj.Matches( Content );
foreach ( Match match in matches )
{
if ( VTitleAddOnce ) break;
VolInstruction VInst = null;
if( RegTitle.Valid )
{
string FTitle = string.Format(
RegTitle.Format
, match.Groups
.Cast<Group>()
.Select( g => g.Value )
.ToArray()
);
if( string.IsNullOrEmpty( RegTitle.Pattern ) )
{
VTitleAddOnce = true;
}
VInst = new VolInstruction(
VTitleAddOnce ? SpTOC.LastIndex : match.Index
, FTitle.ToCTrad()
);
VInst.ProcMan = VolProcs;
SpTOC.PushInstruction( VInst );
}
else
{
ProcManager.PanelMessage( this, () => Res.RSTR( "InvalidPattern" ), LogType.WARNING );
continue;
}
if ( HasVolProcs && RegParam.Valid )
{
string FParam = string.Format(
RegParam.Format
, match.Groups
.Cast<Group>()
.Select( g => g.Value )
.ToArray()
);
if( VolAsync )
{
VInst.SetProcId( VolProcs.GUID );
VInst.PushConvoyParam( FParam );
}
else
{
ProcConvoy VolConvoy = await VolProcs.CreateSpider().Crawl( new ProcConvoy( PPass, VInst ) );
}
}
}
RegTitle = new ProcFind.RegItem( EpPattern, EpTitle, true );
RegParam = new ProcFind.RegItem( EpPattern, EpParam, true );
if ( !( RegTitle.Validate() || RegParam.Validate() ) ) continue;
matches = RegTitle.RegExObj.Matches( Content );
foreach ( Match match in matches )
{
EpInstruction EInst = null;
if( RegTitle.Valid )
{
string FTitle = string.Format(
RegTitle.Format
, match.Groups
.Cast<Group>()
.Select( g => g.Value )
.ToArray()
);
EInst = new EpInstruction(
VTitleAddOnce ? SpTOC.LastIndex : match.Index
, FTitle.ToCTrad()
);
EInst.ProcMan = EpProcs;
SpTOC.PushInstruction( EInst );
}
else
{
ProcManager.PanelMessage( this, () => Res.RSTR( "InvalidPattern" ), LogType.WARNING );
continue;
}
if ( HasEpProcs && RegParam.Valid )
{
string FParam = string.Format(
RegParam.Format
, match.Groups
.Cast<Group>()
.Select( g => g.Value )
.ToArray()
);
if( EpAsync )
{
EInst.SetProcId( EpProcs.GUID );
EInst.PushConvoyParam( FParam );
}
else
{
ProcConvoy EpConvoy = await EpProcs.CreateSpider().Crawl( new ProcConvoy( PPass, FParam ) );
}
}
}
}
return new ProcConvoy( this, SpTOC );
}
public override async Task<ProcConvoy> Run( ProcConvoy Convoy )
{
Convoy = await base.Run( Convoy );
ProcConvoy UsableConvoy;
if ( !TryGetConvoy(
out UsableConvoy
, ( P, C ) =>
C.Payload is IEnumerable<IStorageFile>
|| C.Payload is IEnumerable<string>
|| C.Payload is IStorageFile
|| C.Payload is string
) ) return Convoy;
List<BookInstruction> SpItemList = null;
// Search for the closest Instruction Set
ProcConvoy SpiderInst = ProcManager.TracePackage(
Convoy
, ( P, C ) => C.Payload is IEnumerable<BookInstruction>
);
if ( SpiderInst != null )
{
SpItemList = ( List<BookInstruction> ) SpiderInst.Payload;
}
if ( SpItemList == null )
{
SpItemList = new List<BookInstruction>();
}
ProcPassThru PPass = new ProcPassThru( new ProcConvoy( this, SpItemList ) );
ProcConvoy KnownBook = ProcManager.TracePackage( Convoy, ( P, C ) => C.Payload is BookInstruction );
if ( UsableConvoy.Payload is IEnumerable<IStorageFile> )
{
IEnumerable<IStorageFile> ISFs = ( IEnumerable<IStorageFile> ) UsableConvoy.Payload;
foreach ( IStorageFile ISF in ISFs )
{
string Content = await ISF.ReadString();
await SearchBooks( SpItemList, PPass, KnownBook, Content );
}
}
else if ( UsableConvoy.Payload is IEnumerable<string> )
{
IEnumerable<string> Contents = ( IEnumerable<string> ) UsableConvoy.Payload;
foreach ( string Content in Contents )
{
await SearchBooks( SpItemList, PPass, KnownBook, Content );
}
}
else if ( UsableConvoy.Payload is IStorageFile )
{
IStorageFile ISF = ( IStorageFile ) UsableConvoy.Payload;
string Content = await ISF.ReadString();
await SearchBooks( SpItemList, PPass, KnownBook, Content );
}
else // string
{
await SearchBooks( SpItemList, PPass, KnownBook, ( string ) UsableConvoy.Payload );
}
return new ProcConvoy( this, SpItemList );
}