/// <summary> /// Checks if a Url belongs to the part of the web we wish to crawl. /// </summary> /// <param name="targetUrl">The url to examine</param> /// <returns> /// A <see cref="DomainFlagValue"/> indicating whether the input Url belongs to the /// part of the web we wish to crawl. /// </returns> /// <remarks> /// Since it is possible for a url that belongs to a non-greek domain (e.g. .com) to /// contain greek content, all Urls that do not belong to the .gr domain will get the /// value of <see cref="DomainFlagValue.Unknown"/>. This allows the system to assign /// them at a later time to a client who will visit them and check their content-type /// encoding, in order to determine whether they are of interest to the system. /// </remarks> public DomainFlagValue FilterUrl(ref string targetUrl) { DomainFlagValue retVal = DomainFlagValue.Unknown; try { mutex.WaitOne(); string targetHost = InternetUtils.HostName(targetUrl); if (targetHost.EndsWith(".gr") || targetHost.Contains("ebay.com")) { retVal = DomainFlagValue.MustVisit; } else { if (IsIPAddress(ref targetHost)) { if (ipTable.GetCountry(targetHost) == "GR") { retVal = DomainFlagValue.MustVisit; } } } } catch (Exception e) { globals.FileLog.LogWarning(e.ToString()); } finally { mutex.ReleaseMutex(); } return(retVal); }
/// <summary> /// Constructs a new <see cref="InternetUrlToIndex"/> object from an existing <see cref="InternetUrlToIndex"/> object. /// </summary> /// <param name="IUrl">The existing <see cref="InternetUrlToIndex"/> object</param> public InternetUrlToIndex(InternetUrlToIndex IUrl) { ID = IUrl.ID; Url = IUrl.Url; //m_UrlMD5=IUrlVal.MD5; flagRobots = IUrl.FlagRobots; flagDomain = IUrl.FlagDomain; priority = IUrl.Priority; }
/// <summary> /// Constructs a new <see cref="InternetUrlToCrawl"/> object from an existing <see cref="InternetUrlToCrawl"/> object. /// </summary> /// <param name="IUrl">The existing <see cref="InternetUrlToCrawl"/> object</param> public InternetUrlToCrawl(InternetUrlToCrawl IUrl) { ID = IUrl.ID; Url = IUrl.Url; //m_UrlMD5=IUrlVal.MD5; crc = IUrl.CRC; flagDomain = IUrl.FlagDomain; flagFetchRobots = IUrl.FlagFetchRobots; robotsDisallowedPaths = IUrl.RobotsDisallowedPaths; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToCrawl"/> class. /// </summary> public InternetUrlToCrawl() { ID = 0; //Url=String.Empty; //MD5 = new byte[16]{0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; crc = 0L; flagDomain = DomainFlagValue.Unknown; flagFetchRobots = false; robotsDisallowedPaths = String.Empty; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrl"/> class. /// </summary> public InternetUrlToIndex() { ID = 0; Url = String.Empty; MD5 = new byte[16] { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; flagRobots = false; flagDomain = DomainFlagValue.MustVisit; priority = 255; }
/// <summary> /// Constructs an instance of the <see cref="UrlCrawlData"/> class and initializes /// it with the default values. /// </summary> public UrlCrawlData() { url = new InternetUrlToCrawl(); updated = false; redirected = false; redirectedFlagRobots = false; redirectedFlagDomain = DomainFlagValue.MustVisit; redirectedPriority = 255; httpStatusCode = HttpStatusCode.OK; data = String.Empty; timeStamp = DateTime.UtcNow; retrievalTime = 0; outLinks = null;//new ArrayList(); }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToIndex"/> class that /// points to a given Url and is associated with a given Identifier. All the /// parameters of the newly created instance are supplied and initialized. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrlToIndex"/></param> /// <param name="url">The Url that this object encapsulates</param> /// <param name="md5">The MD5 Hash Code for this <see cref="InternetUrlToIndex"/> object</param> /// <param name="flagRobots">A boolean flag indicating whether it's allowed to visit the Url</param> /// <param name="flagDomain">A <see cref="DomainFlagValue"/> flag indicating whether the Url belongs to the domain we wish to visit</param> /// <param name="priority">A <see cref="System.Byte"/> flag indicating the url's crawling priority</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Url. /// </exception> public InternetUrlToIndex(int ID, string url, byte[] md5, bool flagRobots, DomainFlagValue flagDomain, byte priority) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } MD5 = md5; this.flagRobots = flagRobots; this.flagDomain = flagDomain; this.priority = priority; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToIndex"/> class that /// points to a given Url and is associated with a given Identifier. Assumes that /// it's allowed to visit this Url (the <see cref="FlagRobots"/> property is set /// to false) and that it belongs to the domain we wish to visit. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrlToIndex"/></param> /// <param name="url">The Url that this object encapsulates</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Url. /// </exception> public InternetUrlToIndex(int ID, string url) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } //m_UrlMD5=MD5Hash.md5(UrlVal); flagRobots = false; flagDomain = DomainFlagValue.MustVisit; priority = 255; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToCrawl"/> class that /// points to a given Url and is associated with a given Identifier. All of the /// parameters of the newly created instance are supplied and initialized. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrl"/>.</param> /// <param name="url">The Url that this object encapsulates.</param> /// <param name="md5">The MD5 Hash Code for this <see cref="InternetUrl"/> object.</param> /// <param name="crc">The Cyclic Redundancy Check value associated with the <see cref="InternetUrl"/> object.</param> /// <param name="flagDomain">A <see cref="DomainFlagValue"/> flag indicating whether the crawler must check the country of origin of the contents of the Url.</param> /// <param name="flagFetchRobots">A boolean flag indicating whether a new copy of the robots.txt file related with this Url must be downloaded.</param> /// <param name="robotsDisallowedPaths">A string containing the space-delimited paths on a host that the crawler must not visit.</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Uri. /// </exception> public InternetUrlToCrawl(int ID, string url, byte[] md5, long crc, DomainFlagValue flagDomain, bool flagFetchRobots, string robotsDisallowedPaths) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } MD5 = md5; this.crc = crc; this.flagDomain = flagDomain; this.flagFetchRobots = flagFetchRobots; this.robotsDisallowedPaths = robotsDisallowedPaths; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToCrawl"/> class that /// points to a given Url and is associated with a given Identifier. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrlToCrawl"/>.</param> /// <param name="url">The Url that this object encapsulates.</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Uri. /// </exception> public InternetUrlToCrawl(int ID, string url) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } //m_UrlMD5=MD5Hash.md5(UrlVal); crc = 0L; flagDomain = DomainFlagValue.Unknown; flagFetchRobots = false; robotsDisallowedPaths = String.Empty; }
/// <summary> /// Attempts to extract the appropriate FlagDomain value from the contents of the document. /// </summary> /// <param name="content">The HTML content that must be parsed for Domain Flag value.</param> /// <returns>A <see cref="DomainFlagValue"/> indicating whether the content of the /// text document is in the language that interests us. /// </returns> private DomainFlagValue ExtractDomainFlag(ref string content) { DomainFlagValue retVal = DomainFlagValue.Unknown; try { //attempt to find any greek character in the contents of the document. if (content.IndexOfAny(greekChars) != -1) { //a greek character was found, assume the content is in greek retVal = DomainFlagValue.MustVisit; } else { retVal = DomainFlagValue.MustNotVisit; } } catch {} return(retVal); }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToIndex"/> class that /// points to a given Url and is associated with a given Identifier. All the /// parameters of the newly created instance are supplied and initialized. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrlToIndex"/></param> /// <param name="url">The Url that this object encapsulates</param> /// <param name="md5">The MD5 Hash Code for this <see cref="InternetUrlToIndex"/> object</param> /// <param name="flagRobots">A boolean flag indicating whether it's allowed to visit the Url</param> /// <param name="flagDomain">A <see cref="DomainFlagValue"/> flag indicating whether the Url belongs to the domain we wish to visit</param> /// <param name="priority">A <see cref="System.Byte"/> flag indicating the url's crawling priority</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Url. /// </exception> public InternetUrlToIndex(int ID, string url, byte[] md5, bool flagRobots, DomainFlagValue flagDomain, byte priority) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } MD5 = md5; this.flagRobots = flagRobots; this.flagDomain = flagDomain; this.priority = priority; }
/// <summary> /// Constructs a new <see cref="InternetUrlToIndex"/> object from an existing <see cref="InternetUrlToIndex"/> object. /// </summary> /// <param name="IUrl">The existing <see cref="InternetUrlToIndex"/> object</param> public InternetUrlToIndex(InternetUrlToIndex IUrl) { ID = IUrl.ID; Url = IUrl.Url; //m_UrlMD5=IUrlVal.MD5; flagRobots = IUrl.FlagRobots; flagDomain = IUrl.FlagDomain; priority = IUrl.Priority; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrl"/> class. /// </summary> public InternetUrlToIndex() { ID = 0; Url = String.Empty; MD5 = new byte[16] { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; flagRobots = false; flagDomain = DomainFlagValue.MustVisit; priority = 255; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToIndex"/> class that /// points to a given Url and is associated with a given Identifier. Assumes that /// it's allowed to visit this Url (the <see cref="FlagRobots"/> property is set /// to false) and that it belongs to the domain we wish to visit. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrlToIndex"/></param> /// <param name="url">The Url that this object encapsulates</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Url. /// </exception> public InternetUrlToIndex(int ID, string url) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } //m_UrlMD5=MD5Hash.md5(UrlVal); flagRobots = false; flagDomain = DomainFlagValue.MustVisit; priority = 255; }
/// <summary> /// Constructs a new <see cref="InternetUrlToCrawl"/> object from an existing <see cref="InternetUrlToCrawl"/> object. /// </summary> /// <param name="IUrl">The existing <see cref="InternetUrlToCrawl"/> object</param> public InternetUrlToCrawl(InternetUrlToCrawl IUrl) { ID = IUrl.ID; Url = IUrl.Url; //m_UrlMD5=IUrlVal.MD5; crc = IUrl.CRC; flagDomain = IUrl.FlagDomain; flagFetchRobots = IUrl.FlagFetchRobots; robotsDisallowedPaths = IUrl.RobotsDisallowedPaths; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToCrawl"/> class that /// points to a given Url and is associated with a given Identifier. All of the /// parameters of the newly created instance are supplied and initialized. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrl"/>.</param> /// <param name="url">The Url that this object encapsulates.</param> /// <param name="md5">The MD5 Hash Code for this <see cref="InternetUrl"/> object.</param> /// <param name="crc">The Cyclic Redundancy Check value associated with the <see cref="InternetUrl"/> object.</param> /// <param name="flagDomain">A <see cref="DomainFlagValue"/> flag indicating whether the crawler must check the country of origin of the contents of the Url.</param> /// <param name="flagFetchRobots">A boolean flag indicating whether a new copy of the robots.txt file related with this Url must be downloaded.</param> /// <param name="robotsDisallowedPaths">A string containing the space-delimited paths on a host that the crawler must not visit.</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Uri. /// </exception> public InternetUrlToCrawl(int ID, string url, byte[] md5, long crc, DomainFlagValue flagDomain, bool flagFetchRobots, string robotsDisallowedPaths) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } MD5 = md5; this.crc = crc; this.flagDomain = flagDomain; this.flagFetchRobots = flagFetchRobots; this.robotsDisallowedPaths = robotsDisallowedPaths; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToCrawl"/> class that /// points to a given Url and is associated with a given Identifier. /// </summary> /// <param name="ID">The unique identifier associated with this <see cref="InternetUrlToCrawl"/>.</param> /// <param name="url">The Url that this object encapsulates.</param> /// <exception cref="CWMalformedUrlException"> /// Thrown if the supplied string value is not a valid Uri. /// </exception> public InternetUrlToCrawl(int ID, string url) { Url = url; //this may throw an exception this.ID = ID; // try // { // Uri tmp = new Uri(UrlVal); //this may throw an exception // Url = UrlVal; // } // catch // { // throw new CWMalformedUrlException("The address " + UrlVal + " is not a valid Uri."); // } //m_UrlMD5=MD5Hash.md5(UrlVal); crc = 0L; flagDomain = DomainFlagValue.Unknown; flagFetchRobots = false; robotsDisallowedPaths = String.Empty; }
/// <summary> /// Constructs a new instance of the <see cref="InternetUrlToCrawl"/> class. /// </summary> public InternetUrlToCrawl() { ID = 0; //Url=String.Empty; //MD5 = new byte[16]{0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; crc = 0L; flagDomain = DomainFlagValue.Unknown; flagFetchRobots = false; robotsDisallowedPaths = String.Empty; }
/// <summary> /// Constructs an instance of the <see cref="UrlCrawlData"/> class and initializes /// it with the default values. /// </summary> public UrlCrawlData() { url=new InternetUrlToCrawl(); updated=false; redirected=false; redirectedFlagRobots = false; redirectedFlagDomain = DomainFlagValue.MustVisit; redirectedPriority = 255; httpStatusCode=HttpStatusCode.OK; data=String.Empty; timeStamp=DateTime.UtcNow; retrievalTime=0; outLinks=null;//new ArrayList(); }