public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider)
{
UrlType urlType = ((WebSpiderResource)parentSpider).addingUrlType;
if (
!(urlType == UrlType.Img && this.downloadImagesFromExternalSite)
&&
!(urlType != UrlType.Img && this.downloadResourceFromExternalSite)
)
{
string tempbase = context.BaseLocation;
int i;
if ((i = tempbase.LastIndexOf("/")) >= 0)
{
tempbase = tempbase.Substring(0, i + 1);
}
if (!location.OriginalString.StartsWith(tempbase))
{
return(false);
}
}
return(true);
}
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider)
{
UrlType urlType = ((WebSpiderResource)parentSpider).addingUrlType;
if (
!(urlType == UrlType.Img && this.downloadImagesFromExternalSite)
&&
!(urlType != UrlType.Img && this.downloadResourceFromExternalSite)
)
{
string tempbase = context.BaseLocation;
int i;
if ((i = tempbase.LastIndexOf("/")) >= 0)
{
tempbase = tempbase.Substring(0, i + 1);
}
if (!location.OriginalString.StartsWith(tempbase))
{
return false;
}
}
return true;
}
protected internal virtual void OnResourceCompleted(ISpiderResource spider)
{
if (ResourceCompleted != null)
{
ResourceCompleted(this, spider);
}
}
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider)
{
string ext = Path.GetExtension(location.Segments[location.Segments.Length - 1]).ToLower();
int index = Array.IndexOf <string>(this.extensionsToExclude, ext);
return(index < 0);
}
public virtual IAsyncRetriver BeginLookup()
{
ISpiderResource resourceSpider = factory.CreateSpider(this.context, null, this.context.BaseLocation);
context.AddURIToVisit(new Uri(this.context.BaseLocation), resourceSpider);
return(resourceSpider.BeginReceive());
}
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider)
{
string ext = Path.GetExtension(location.Segments[location.Segments.Length-1]).ToLower();
int index = Array.IndexOf<string>(this.extensionsToExclude, ext);
return index < 0;
}
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider)
{
if (this.maxDepth > 0 && parentSpider.Depth >= this.maxDepth)
{
return(false);
}
else
{
return(true);
}
}
public bool Accept(Uri location, SpiderContext context, ISpiderResource parentSpider)
{
if (this.maxDepth > 0 && parentSpider.Depth >= this.maxDepth)
{
return false;
}
else
{
return true;
}
}
public bool Accept(Uri uri, SpiderContext context, ISpiderResource parentSpider)
{
string fileName = uri.Segments[uri.Segments.Length - 1];
for (int i = 0; i < namesToExclude.Length; i++)
{
if (fileName.IndexOf(namesToExclude[i]) >= 0)
{
return(false);
}
}
return(true);
}
public bool Accept(Uri uri, SpiderContext context, ISpiderResource parentSpider)
{
string fileName = uri.Segments[uri.Segments.Length - 1];
for (int i = 0; i < namesToExclude.Length; i++)
{
if (fileName.IndexOf(namesToExclude[i]) >= 0)
{
return false;
}
}
return true;
}
private void UpdateDepth()
{
if (this.parent == null)
{
this.depth = 0;
}
else
{
ISpiderResource tmpParent = this.parent;
while (tmpParent != null)
{
this.depth++;
tmpParent = tmpParent.ParentResource;
}
}
}
public void NotifyCompletion(ISpiderResource spider)
{
lock (completedResources)
{
if (!completedResources.Contains(spider))
{
completedResources.Add(spider);
}
}
OnResourceCompleted(spider);
if (completedResources.Count == VisitedLocations.Count)
{
OnCompleted();
}
}
public WebSpiderResource(SpiderContext context, ISpiderResource parent, string location)
{
this.context = context;
this.location = location;
this.parent = parent;
UpdateDepth();
string localFile = GetLocalFile();
download = DownloadManager.Instance.Add(
ResourceLocation.FromURL(this.Location),
null,
localFile,
1,
false);
download.StateChanged += new EventHandler(download_StateChanged);
}
public WebSpiderResource(SpiderContext context, ISpiderResource parent, string location)
{
this.context = context;
this.location = location;
this.parent = parent;
UpdateDepth();
string localFile = GetLocalFile();
download = DownloadManager.Instance.Add(
ResourceLocation.FromURL(this.Location),
null,
localFile,
1,
false);
download.StateChanged += new EventHandler(download_StateChanged);
}
public bool AddURIToVisit(Uri uri, ISpiderResource parentSpider)
{
if (ShouldAdd(uri, parentSpider))
{
lock (visitedLocations)
{
if (visitedLocations.Contains(uri))
{
return(false);
}
visitedLocations.Add(uri);
return(true);
}
}
else
{
return(false);
}
}
private bool ShouldAdd(Uri location, ISpiderResource parentSpider)
{
#region Aborted
if (aborted)
{
return(false);
}
#endregion
#region Naming rules
for (int j = 0; j < namingRules.Count; j++)
{
if (!namingRules[j].Accept(location, this, parentSpider))
{
return(false);
}
}
#endregion
return(true);
}
protected internal virtual void OnResourceCompleted(ISpiderResource spider)
{
if (ResourceCompleted != null)
{
ResourceCompleted(this, spider);
}
}
public ISpiderResource CreateSpider(SpiderContext cntx, ISpiderResource parent, string location)
{
return(new WebSpiderResource(cntx, parent, location));
}
public ISpiderResource CreateSpider(SpiderContext cntx, ISpiderResource parent, string location)
{
return new WebSpiderResource(cntx, parent, location);
}
internal DownloaderAsyncRetrive(ISpiderResource resource, Downloader d)
{
this.downloader = d;
this.resource = resource;
}
public bool AddURIToVisit(Uri uri, ISpiderResource parentSpider)
{
if (ShouldAdd(uri, parentSpider))
{
lock (visitedLocations)
{
if (visitedLocations.Contains(uri))
{
return false;
}
visitedLocations.Add(uri);
return true;
}
}
else
{
return false;
}
}
public void NotifyCompletion(ISpiderResource spider)
{
lock (completedResources)
{
if (!completedResources.Contains(spider))
{
completedResources.Add(spider);
}
}
OnResourceCompleted(spider);
if (completedResources.Count == VisitedLocations.Count)
{
OnCompleted();
}
}
private bool ShouldAdd(Uri location, ISpiderResource parentSpider)
{
#region Aborted
if (aborted)
{
return false;
}
#endregion
#region Naming rules
for (int j = 0; j < namingRules.Count; j++)
{
if (!namingRules[j].Accept(location, this, parentSpider)) return false;
}
#endregion
return true;
}
internal DownloaderAsyncRetrive(ISpiderResource resource, Downloader d)
{
this.downloader = d;
this.resource = resource;
}
