コード例 #1
0
ファイル: UserAgentEntry.cs プロジェクト: xavivars/nrobots
        public void AddEntry(Entry entry)
        {
            if (entry == null)
                throw new ArgumentNullException("entry");

            _entries.Add(entry);
        }
コード例 #2
0
ファイル: Robots.cs プロジェクト: xavivars/nrobots
        internal void AddEntry(Entry entry)
        {
            if (entry ==  null)
                throw new ArgumentNullException("entry");
            if (entry.Type != EntryType.Comment && entry.Type != EntryType.UserAgent)
                throw new ArgumentException("Only Comments and User-Agent entries can be added", "entry");

            _entries.Add(entry);
        }
コード例 #3
0
ファイル: Entry.cs プロジェクト: xavivars/nrobots
        public static bool TryParse(Uri baseUri, string entryText, out Entry entry)
        {
            if (baseUri == null)
                throw new ArgumentNullException("baseUri");


            entry = null;

            if (string.IsNullOrEmpty(entryText))
                return false;

            EntryType type = EntryType.Invalid;
            string comment = string.Empty;

            int commentPosition = entryText.IndexOf('#');
            if (commentPosition == 0)
            {
                type = EntryType.Comment;
            }
            if (commentPosition >= 0 && 
                (commentPosition == 0 || entryText[commentPosition - 1] == ' '))
            {
                comment = entryText.Substring(commentPosition + 1);
                entryText = entryText.Substring(0, commentPosition);
            }

            if (string.IsNullOrEmpty(entryText))
            {
                entry = CreateEntry(type);
                entry.Comment = comment;
                return true;
            }
            try
            {
                if (entryText.StartsWith(USER_AGENT_KEYWORD, true, CultureInfo.InvariantCulture))
                {
                    type = EntryType.UserAgent;
                    entry = CreateEntry(type);
                    entry.Comment = comment;
                    string userAgent = entryText.Substring(USER_AGENT_KEYWORD.Length).Trim().TrimEnd('?');;

                    if(userAgent == null || string.IsNullOrEmpty(userAgent.Trim()))
                    {
                        entry = null;
                        return false;                         
                    }
                    else
                    {
                         ((UserAgentEntry) entry).UserAgent = userAgent;
                    }
                }
                else if (entryText.StartsWith(DISALLOW_KEYWORD, true, CultureInfo.InvariantCulture))
                {
                    type = EntryType.Disallow;
                    bool inverted = entryText.EndsWith("$");
                    string value = entryText.Substring(DISALLOW_KEYWORD.Length).Trim().TrimEnd('?');

                    Uri url = null;
                    if (!string.IsNullOrEmpty(value) && Uri.TryCreate(baseUri, value, out url))
                    {
                        entry = CreateEntry(type);
                        entry.Comment = comment;
                        ((UrlEntry)entry).Url = url;
                        ((UrlEntry)entry).Inverted = inverted;
                    }
                }
                else if (entryText.StartsWith(ALLOW_KEYWORD, true, CultureInfo.InvariantCulture))
                {
                    type = EntryType.Allow;
                    bool inverted = entryText.EndsWith("$");
                    string value = entryText.Substring(ALLOW_KEYWORD.Length).Trim().TrimEnd('?');

                    Uri url;
                    if (Uri.TryCreate(baseUri, value, out url))
                    {
                        entry = CreateEntry(type);
                        entry.Comment = comment;
                        ((UrlEntry)entry).Url = url;
                        ((UrlEntry)entry).Inverted = inverted;
                    }
                }
                else if (entryText.StartsWith(CRAWL_DELAY_KEYWORD, true, CultureInfo.InvariantCulture))
                {
                    type = EntryType.CrawlDelay;
                    string value = entryText.Substring(CRAWL_DELAY_KEYWORD.Length).Trim().TrimEnd('?');

                    entry = CreateEntry(type);
                    entry.Comment = comment;

                    try
                    {
                        ((CrawlDelayEntry)entry).CrawlDelay = Convert.ToInt32(value);
                    }
                    catch
                    {
                        ((CrawlDelayEntry)entry).CrawlDelay = 0;
                    }
                }
                else if (entryText.StartsWith(SITEMAP_KEYWORD, true, CultureInfo.InvariantCulture))
                {
                    type = EntryType.Sitemap;
                    string value = entryText.Substring(SITEMAP_KEYWORD.Length).Trim().TrimEnd('?');

                    entry = CreateEntry(type);
                    entry.Comment = comment;

                    try
                    {
                        ((SitemapEntry)entry).SitemapUrl = value;
                    }
                    catch
                    {
                        ((SitemapEntry)entry).SitemapUrl = "";
                    }
                }
                else
                {
                    entry = null;
                    return false;
                }
            }
            catch (Exception ex)
            {
                entry = null;
                return false;
            }
            if (entry == null)
                return false;
            return true;
        }
コード例 #4
0
ファイル: Entry.cs プロジェクト: Kemyke/nrobots-dotnet-core
        public static bool TryParse(Uri baseUri, string entryText, out Entry entry)
        {
            if (baseUri == null)
            {
                throw new ArgumentNullException("baseUri");
            }


            entry = null;

            if (string.IsNullOrEmpty(entryText))
            {
                return(false);
            }

            EntryType type    = EntryType.Invalid;
            string    comment = string.Empty;

            int commentPosition = entryText.IndexOf('#');

            if (commentPosition == 0)
            {
                type = EntryType.Comment;
            }
            if (commentPosition >= 0 &&
                (commentPosition == 0 || entryText[commentPosition - 1] == ' '))
            {
                comment   = entryText.Substring(commentPosition + 1);
                entryText = entryText.Substring(0, commentPosition);
            }

            if (string.IsNullOrEmpty(entryText))
            {
                entry         = CreateEntry(type);
                entry.Comment = comment;
                return(true);
            }
            try
            {
                if (entryText.StartsWith(USER_AGENT_KEYWORD, StringComparison.OrdinalIgnoreCase))
                {
                    type          = EntryType.UserAgent;
                    entry         = CreateEntry(type);
                    entry.Comment = comment;
                    string userAgent = entryText.Substring(USER_AGENT_KEYWORD.Length).Trim().TrimEnd('?');

                    if (userAgent == null || string.IsNullOrEmpty(userAgent.Trim()))
                    {
                        entry = null;
                        return(false);
                    }
                    else
                    {
                        ((UserAgentEntry)entry).UserAgent = userAgent;
                    }
                }
                else if (entryText.StartsWith(DISALLOW_KEYWORD, StringComparison.OrdinalIgnoreCase))
                {
                    type = EntryType.Disallow;
                    bool   inverted = entryText.EndsWith("$");
                    string value    = entryText.Substring(DISALLOW_KEYWORD.Length).Trim();

                    Uri url = null;
                    if (!string.IsNullOrEmpty(value) && Uri.TryCreate(baseUri, value, out url))
                    {
                        entry                      = CreateEntry(type);
                        entry.Comment              = comment;
                        ((UrlEntry)entry).Url      = url;
                        ((UrlEntry)entry).Inverted = inverted;
                    }
                }
                else if (entryText.StartsWith(ALLOW_KEYWORD, StringComparison.OrdinalIgnoreCase))
                {
                    type = EntryType.Allow;
                    bool   inverted = entryText.EndsWith("$");
                    string value    = entryText.Substring(ALLOW_KEYWORD.Length).Trim().TrimEnd('?');

                    Uri url;
                    if (Uri.TryCreate(baseUri, value, out url))
                    {
                        entry                      = CreateEntry(type);
                        entry.Comment              = comment;
                        ((UrlEntry)entry).Url      = url;
                        ((UrlEntry)entry).Inverted = inverted;
                    }
                }
                else if (entryText.StartsWith(CRAWL_DELAY_KEYWORD, StringComparison.OrdinalIgnoreCase))
                {
                    type = EntryType.CrawlDelay;
                    string value = entryText.Substring(CRAWL_DELAY_KEYWORD.Length).Trim().TrimEnd('?');

                    entry         = CreateEntry(type);
                    entry.Comment = comment;

                    try
                    {
                        ((CrawlDelayEntry)entry).CrawlDelay = Convert.ToInt32(value);
                    }
                    catch
                    {
                        ((CrawlDelayEntry)entry).CrawlDelay = 0;
                    }
                }
                else if (entryText.StartsWith(SITEMAP_KEYWORD, StringComparison.OrdinalIgnoreCase))
                {
                    type = EntryType.Sitemap;
                    string value = entryText.Substring(SITEMAP_KEYWORD.Length).Trim().TrimEnd('?');

                    entry         = CreateEntry(type);
                    entry.Comment = comment;

                    try
                    {
                        ((SitemapEntry)entry).SitemapUrl = value;
                    }
                    catch
                    {
                        ((SitemapEntry)entry).SitemapUrl = "";
                    }
                }
                else
                {
                    entry = null;
                    return(false);
                }
            }
            catch (Exception)
            {
                entry = null;
                return(false);
            }
            if (entry == null)
            {
                return(false);
            }
            return(true);
        }