/// <summary>
        /// Called when the spider is ready to process an HTML
        /// URL. Download the contents of the URL to a local file.
        /// </summary>
        /// <param name="url">The URL that the spider is about to process.</param>
        /// <param name="parse">An object that will allow you you to parse the HTML on this page.</param>
        public void SpiderProcessURL(Uri url, SpiderParseHTML parse)
        {
            String filename = URLUtility.convertFilename(this.path, url, true);
            Stream os       = new FileStream(filename, FileMode.Create);

            parse.Stream.OutputStream = os;
            parse.ReadAll();
            os.Close();
        }
        public void FindBestIcon()
        {
            if (this.SiteUri == null)
            {
                this.myLogger.LogWarn("No URL. Stop Find Best Icon");
                return;
            }

            FindBestIconForURI(this.SiteUri);

            if (this.BestImage == null)               // try with www

            {
                Uri testURI = this.SiteUri;
                if (!testURI.Host.StartsWith("www."))
                {
                    FindBestIconForURI(new Uri(
                                           this.SiteUri.Scheme + "://www." + this.SiteUri.DnsSafeHost));
                }
            }

            string rootDomain = URLUtility.RootDomain(this.SiteUri.AbsoluteUri);

            if (!string.IsNullOrEmpty(rootDomain))
            {
                if (this.BestImage == null)                   // try with root domain prefix
                {
                    FindBestIconForURI(new Uri(this.SiteUri.Scheme + "://" + rootDomain));
                }

                if (this.BestImage == null)                   // try with host prefix and www
                {
                    FindBestIconForURI(new Uri(this.SiteUri.Scheme + "://www." + rootDomain));
                }
            }

            if (this.RootUriForIco != null)
            {
                if (this.BestImage == null)                   // try with host prefix
                {
                    FindBestIconForURI(new Uri(
                                           this.RootUriForIco.Scheme + "://" + this.RootUriForIco.Host));
                }

                if (this.BestImage == null)                   // try with host prefix and www
                {
                    FindBestIconForURI(new Uri(
                                           this.RootUriForIco.Scheme + "://www." + this.RootUriForIco.Host));
                }
            }

            if (this.BestImage == null)
            {
                FindBestIconForURI(this.OriginalSiteUri);
            }
        }
Exemple #3
0
 public void TestIsValidURL()
 {
     Assert.That(URLUtility.isValidURLString("http://github.com/vitdeck/VitDeck"), Is.True);
     Assert.That(URLUtility.isValidURLString("https://github.com/vitdeck/VitDeck/blob/56b11a7dbbca3b8cf6c4104ca443f78f2beae9c1/LICENSE?query=test#1"), Is.True);
     Assert.That(URLUtility.isValidURLString("http://www.google.co.jp/search?q=C"), Is.True);
     Assert.That(URLUtility.isValidURLString("ftp://github.com/"), Is.False);
     Assert.That(URLUtility.isValidURLString(""), Is.False);
     Assert.That(URLUtility.isValidURLString(null), Is.False);
     Assert.That(URLUtility.isValidURLString("Test String"), Is.False);
 }
Exemple #4
0
        public static string ComposeURL(Protocol protocol, bool secure, string host, int port)
        {
            string result = URLUtility.ProtocolPrefix(protocol, secure) + "://" + host;

            // Make this removal optional
            if (port != 80)
            {
                result += ":" + port;
            }

            return(result);
        }
        /// <summary>
        /// Called when the spider is about to process a NON-HTML
        /// URL.
        /// </summary>
        /// <param name="url">The URL that the spider found.</param>
        /// <param name="stream">An InputStream to read the page contents from.</param>
        public void SpiderProcessURL(Uri url, Stream stream)
        {
            byte[] buffer = new byte[1024];

            int    length;
            String filename = URLUtility.convertFilename(this.path, url, true);

            Stream os = new FileStream(filename, FileMode.Create);

            do
            {
                length = stream.Read(buffer, 0, buffer.Length);
                if (length > 0)
                {
                    os.Write(buffer, 0, length);
                }
            } while (length > 0);
            os.Close();
        }
Exemple #6
0
        /// <summary>
        /// Used internally, to add a URL to the spider's workload.
        /// </summary>
        /// <param name="u">The URL to add.</param>
        /// <param name="type">What type of link this is.</param>
        private void AddURL(String u, Spider.URLType type)
        {
            if (u == null)
            {
                return;
            }

            try
            {
                Uri url = URLUtility.constructURL(this.baseURL, u, true);
                url = this.spider.Workload.ConvertURL(url.ToString());

                if ((String.Compare(url.Scheme, "http", true) == 0) ||
                    (String.Compare(url.Scheme, "https", true) == 0))
                {
                    if (this.spider.Report.SpiderFoundURL(url, this.baseURL, type))
                    {
                        try
                        {
                            this.spider.AddURL(url, this.baseURL, this.depth + 1);
                        }
                        catch (WorkloadException e)
                        {
                            throw new IOException(e.Message);
                        }
                    }
                }
            }

            catch (UriFormatException)
            {
                spider.Logging.Log(Logger.Level.INFO, "Malformed URL found:" + u);
            }
            catch (WorkloadException)
            {
                spider.Logging.Log(Logger.Level.INFO, "Invalid URL found:" + u);
            }
        }
Exemple #7
0
        /// <summary>
        /// This method is called when an anchor(A) tag is found.
        /// </summary>
        /// <param name="href">The link found.</param>
        private void HandleA(String href)
        {
            String cmp = null;

            if (href != null)
            {
                href = href.Trim();
                cmp  = href.ToLower();
            }


            if ((cmp != null) && !URLUtility.containsInvalidURLCharacters(href))
            {
                if (!cmp.StartsWith("javascript:") &&
                    !cmp.StartsWith("rstp:") &&
                    !cmp.StartsWith("rtsp:") &&
                    !cmp.StartsWith("news:") &&
                    !cmp.StartsWith("irc:") &&
                    !cmp.StartsWith("mailto:"))
                {
                    AddURL(href, Spider.URLType.HYPERLINK);
                }
            }
        }
        public void Initialize(PwDatabase db)
        {
            Debug.Assert(m_dicCustomIconsStats != null); if (m_dicCustomIconsStats == null)
            {
                throw new InvalidOperationException();
            }
            Debug.Assert(m_dicStandardIconsStats != null); if (m_dicStandardIconsStats == null)
            {
                throw new InvalidOperationException();
            }


            GroupHandler gh = delegate(PwGroup pg)
            {
                if (!(pg.CustomIconUuid.Equals(PwUuid.Zero)))
                {
                    if (!(m_dicCustomIconsStats.ContainsKey(pg.CustomIconUuid)))
                    {
                        m_dicCustomIconsStats.Add(pg.CustomIconUuid, new IconStats());
                    }
                    m_dicCustomIconsStats[pg.CustomIconUuid].nbInGroups += 1;
                    m_dicCustomIconsStats[pg.CustomIconUuid].listGroups.Add(pg);
                }
                else
                {
                    if (!(m_dicStandardIconsStats.ContainsKey(pg.IconId)))
                    {
                        m_dicStandardIconsStats.Add(pg.IconId, new IconStats());
                    }
                    m_dicStandardIconsStats[pg.IconId].nbInGroups += 1;
                    m_dicStandardIconsStats[pg.IconId].listGroups.Add(pg);
                }

                return(true);
            };

            String       urlFieldValue;
            Uri          siteUri;
            List <Uri>   uriList;
            EntryHandler eh = delegate(PwEntry pe)
            {
                if (!(pe.CustomIconUuid.Equals(PwUuid.Zero)))
                {
                    if (!(m_dicCustomIconsStats.ContainsKey(pe.CustomIconUuid)))
                    {
                        m_dicCustomIconsStats.Add(pe.CustomIconUuid, new IconStats());
                    }
                    m_dicCustomIconsStats[pe.CustomIconUuid].nbInEntries += 1;
                    m_dicCustomIconsStats[pe.CustomIconUuid].listEntries.Add(pe);
                    uriList = m_dicCustomIconsStats[pe.CustomIconUuid].listUris;
                }
                else
                {
                    if (!(m_dicStandardIconsStats.ContainsKey(pe.IconId)))
                    {
                        m_dicStandardIconsStats.Add(pe.IconId, new IconStats());
                    }
                    m_dicStandardIconsStats[pe.IconId].nbInEntries += 1;
                    m_dicStandardIconsStats[pe.IconId].listEntries.Add(pe);
                    uriList = m_dicStandardIconsStats[pe.IconId].listUris;
                }

                // Get Uri
                urlFieldValue = pe.Strings.ReadSafe(PwDefs.UrlField);
                if (!string.IsNullOrEmpty(urlFieldValue))
                {
                    try {
                        urlFieldValue = URLUtility.addUrlHttpPrefix(urlFieldValue);
                        bool urlWellFFormatted = Uri.TryCreate(urlFieldValue, UriKind.Absolute, out siteUri);

                        if (urlWellFFormatted)
                        {
                            if ((siteUri.Scheme.ToLower() == "http") ||
                                (siteUri.Scheme.ToLower() == "https"))
                            {
                                uriList.Add(siteUri);
                            }
                        }
                    }
                    catch (UriFormatException e)
                    {
                        Debug.Write(String.Format("URIFormatException during parsing uri of entry = {0}", e.Message));
                    }
                    catch (Exception) { };
                }

                return(true);
            };

            gh(db.RootGroup);
            db.RootGroup.TraverseTree(TraversalMethod.PreOrder, gh, eh);

            m_isInitialized = true;
        }
Exemple #9
0
        public string GenerateJS(Type modelType, string host, List <string> readOnlyProperties, List <string> properties, List <string> viewIgnoreProperties, bool hasUpdate, bool hasAdd, bool hasDelete, bool minimize)
        {
            WrappedStringBuilder sb = new WrappedStringBuilder(minimize);

            if (!minimize)
            {
                sb.AppendLine("//Org.Reddragonit.BackBoneDotNet.JSGenerators.ModelListCallGenerators");
            }
            foreach (MethodInfo mi in modelType.GetMethods(Constants.LOAD_METHOD_FLAGS))
            {
                if (mi.GetCustomAttributes(typeof(ModelListMethod), false).Length > 0)
                {
                    foreach (ModelListMethod mlm in mi.GetCustomAttributes(typeof(ModelListMethod), false))
                    {
                        if (mlm.Host == host || mlm.Host == "*")
                        {
                            WrappedStringBuilder sbCurParameters = new WrappedStringBuilder(minimize);
                            sbCurParameters.Append((minimize ? "function(){return{":"function(){return {"));
                            sb.Append(string.Format((minimize ?
                                                     "{0}=_.extend({0},{{{1}:function("
                                : "{0} = _.extend({0}, {{{1}:function("), ModelNamespace.GetFullNameForModel(modelType, host), mi.Name));
                            for (int x = 0; x < (mlm.Paged ? mi.GetParameters().Length - 3 : mi.GetParameters().Length); x++)
                            {
                                sb.Append((x == 0 ? "" : ",") + mi.GetParameters()[x].Name);
                                sbCurParameters.AppendLine((x == 0 ? "" : ",") + string.Format("'{0}':{0}", mi.GetParameters()[x].Name));
                            }
                            sbCurParameters.Append("};}");
                            if (mlm.Paged)
                            {
                                if (mi.GetParameters().Length != 3)
                                {
                                    sb.Append(",");
                                }
                                sb.Append("pageStartIndex,pageSize");
                            }
                            sb.AppendLine("){");
                            string urlCode = URLUtility.CreateJavacriptUrlCode(mlm, mi, modelType);
                            sb.Append(urlCode);
                            if (mlm.Paged)
                            {
                                sb.AppendLine(string.Format((minimize ?
                                                             "pageStartIndex=(pageStartIndex==undefined?0:(pageStartIndex==null?0:pageStartIndex));pageSize=(pageSize==undefined?10:(pageSize==null?10:pageSize));var ret=Backbone.Collection.extend({{url:url+'{0}PageStartIndex='+pageStartIndex+'&PageSize='+pageSize,CurrentParameters:{1},currentIndex:pageStartIndex*pageSize,currentPageSize:pageSize,CurrentPage:Math.floor(pageStartIndex/pageSize),parse:function(response){{if(response.Backbone!=undefined){{_.extend(Backbone,response.Backbone);}}response=response.response;this.TotalPages=response.Pager.TotalPages;return response.response;}},MoveToPage:function(pageNumber){{if(pageNumber>=0&&pageNumber<this.TotalPages){{this.currentIndex=pageNumber*this.currentPageSize;{2}this.fetch();this.CurrentPage=pageNumber;}}}},ChangePageSize:function(pageSize){{this.currentPageSize=pageSize;this.MoveToPage(Math.floor(this.currentIndex/pageSize));}},MoveToNextPage:function(){{if(Math.floor(this.currentIndex/this.currentPageSize)+1<this.TotalPages){{this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)+1);}}}},MoveToPreviousPage:function(){{if(Math.floor(this.currentIndex/this.currentPageSize)-1>=0){{this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)-1);}}}},"
                                    :@"pageStartIndex = (pageStartIndex == undefined ? 0 : (pageStartIndex == null ? 0 : pageStartIndex));
pageSize = (pageSize == undefined ? 10 : (pageSize == null ? 10 : pageSize));
var ret = Backbone.Collection.extend({{url:url+'{0}PageStartIndex='+pageStartIndex+'&PageSize='+pageSize,
    CurrentParameters:{1},
    currentIndex : pageStartIndex*pageSize,
    currentPageSize : pageSize,
    CurrentPage : Math.floor(pageStartIndex/pageSize),
    parse : function(response){{
        if(response.Backbone!=undefined){{
            _.extend(Backbone,response.Backbone);
        }}
        response = response.response;
        this.TotalPages = response.Pager.TotalPages;
        return response.response;
    }},
    MoveToPage : function(pageNumber){{
        if (pageNumber>=0 && pageNumber<this.TotalPages){{
            this.currentIndex = pageNumber*this.currentPageSize;
            {2}
            this.fetch();
            this.CurrentPage=pageNumber;
        }}
    }},
    ChangePageSize : function(pageSize){{
        this.currentPageSize = pageSize;
        this.MoveToPage(Math.floor(this.currentIndex/pageSize));
    }},
    MoveToNextPage : function(){{
        if(Math.floor(this.currentIndex/this.currentPageSize)+1<this.TotalPages){{
            this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)+1);
        }}
    }},
    MoveToPreviousPage : function(){{
        if(Math.floor(this.currentIndex/this.currentPageSize)-1>=0){{
            this.MoveToPage(Math.floor(this.currentIndex/this.currentPageSize)-1);
        }}
    }},"), new object[] {
                                    (mlm.Path.Contains("?") ? "&" : "?"),
                                    sbCurParameters.ToString(),
                                    (mlm.Path.Contains("?") ?
                                     (minimize ? "" : "\t\t\t") + "this.url = this.url.substring(0,this.url.indexOf('&PageStartIndex='))+'&PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;" :
                                     (minimize ? "" : "\t\t\t") + "this.url = this.url.substring(0,this.url.indexOf('?'))+'?PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;")
                                }));
                                if (mi.GetParameters().Length > 0)
                                {
                                    sb.Append((minimize ? "" : "\t") + "ChangeParameters: function(");
                                    for (int x = 0; x < mi.GetParameters().Length - 3; x++)
                                    {
                                        sb.Append((x == 0 ? "" : ",") + mi.GetParameters()[x].Name);
                                    }
                                    sb.AppendLine(string.Format((minimize ?
                                                                 "){{{0}url+='{1}PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;this.CurrentParameters={2};this.currentIndex=0;this.url=url;this.fetch();}},"
                                        :@"){{{0}
        url+='{1}PageStartIndex='+this.currentIndex+'&PageSize='+this.currentPageSize;
        this.CurrentParameters = {2};
        this.currentIndex=0;
        this.url=url;
        this.fetch();
}},"), new object[] { urlCode, (mlm.Path.Contains("?") ? "&" : "?"), sbCurParameters.ToString() }));
                                }
                                sb.AppendLine(string.Format((minimize ?
                                                             "model:{0}.Model}});"
                                    :@" model:{0}.Model
}});"), ModelNamespace.GetFullNameForModel(modelType, host)));
                            }
                            else
                            {
                                sb.AppendLine(string.Format((minimize ?
                                                             "var ret=Backbone.Collection.extend({{url:url,CurrentParameters:{0},parse : function(response){{if(response.Backbone!=undefined){{_.extend(Backbone,response.Backbone);return response.response;}}else{{return response;}}}},"
                                    :@" var ret = Backbone.Collection.extend({{url:url,CurrentParameters:{0},parse : function(response){{
    if(response.Backbone!=undefined){{
        _.extend(Backbone,response.Backbone);
        return response.response;
    }}else{{
        return response;
    }}
}},"), sbCurParameters.ToString()));
                                if (mi.GetParameters().Length > 0)
                                {
                                    sb.Append((minimize ? "" : "\t") + "ChangeParameters: function(");
                                    for (int x = 0; x < mi.GetParameters().Length; x++)
                                    {
                                        sb.Append((x == 0 ? "" : ",") + mi.GetParameters()[x].Name);
                                    }
                                    sb.AppendLine(string.Format((minimize ?
                                                                 "){{{0}this.CurrentParameters={1};this.currentIndex=0;this.url=url;this.fetch();}},"
                                        :@"){{{0}
        this.CurrentParameters = {1};
        this.currentIndex=0;
        this.url=url;
        this.fetch();
}},"), urlCode, sbCurParameters.ToString()));
                                }
                                sb.AppendLine("model:" + ModelNamespace.GetFullNameForModel(modelType, host) + ".Model});");
                            }
                            sb.AppendLine((minimize ?
                                           "ret=new ret();return ret;}});"
                                :@"ret = new ret();
    return ret;
}});"));
                            break;
                        }
                    }
                }
            }
            return(sb.ToString());
        }