Beispiel #1
0
        public string CrawlSite()
        {
            //if i want to crawl the site
            //string sitePrimaryUrl = System.Web.HttpContext.Current.Request.Url.OriginalString;

            string sitePrimaryUrl = "http://myplastikkarten.edigitstore.it/";

            //string sitePrimaryUrl = "https://www.myplastikkarten.de";
            //for main site without any action or controller below section will be skipped
            if (System.Web.HttpContext.Current.Request.Url.PathAndQuery != "/")
            {
                sitePrimaryUrl = sitePrimaryUrl.Replace(System.Web.HttpContext.Current.Request.Url.PathAndQuery, "");
            }

            UriBuilder uri = new UriBuilder("http://myplastikkarten.edigitstore.it/");

            //UriBuilder uri = new UriBuilder("https://www.myplastikkarten.de");
            Crawl c1 = new Crawl();

            c1.PrimaryUrl  = sitePrimaryUrl;
            c1.PrimaryHost = uri.Host;
            c1.GetUrlsOfSite(sitePrimaryUrl);

            List <LocationUrls_Result> lstSItemapResult = new List <LocationUrls_Result>();

            foreach (string singleUrl in Crawl.Urls)
            {
                //qui scartiamo gli url che non sono html o aspx
                Regex           rx      = new Regex(@"^.*\.(?!html$|aspx$)[^.]+$");
                MatchCollection matches = rx.Matches(singleUrl);
                if (matches.Count == 0)
                {
                    lstSItemapResult.Add(new LocationUrls_Result()
                    {
                        Url = singleUrl, Changefreq = "weekly"
                    });
                }
            }
            //escludere ai e psd
            return(GetSitemap(lstSItemapResult));
            //return new XmlSitemapResult(lstSItemapResult);
        }
Beispiel #2
0
        public XmlSitemapResult CrawlSiteGeneratingSitemapOnFly()
        {
            //if i want to crawl the site
            //string sitePrimaryUrl = System.Web.HttpContext.Current.Request.Url.OriginalString;

            string sitePrimaryUrl = "http://myplastikkarten.edigitstore.it/";

            //string sitePrimaryUrl = "https://www.myplastikkarten.de";
            //for main site without any action or controller below section will be skipped
            if (System.Web.HttpContext.Current.Request.Url.PathAndQuery != "/")
            {
                sitePrimaryUrl = sitePrimaryUrl.Replace(System.Web.HttpContext.Current.Request.Url.PathAndQuery, "");
            }

            UriBuilder uri = new UriBuilder("http://myplastikkarten.edigitstore.it/");

            //UriBuilder uri = new UriBuilder("https://www.myplastikkarten.de");
            Crawl c1 = new Crawl();

            c1.PrimaryUrl  = sitePrimaryUrl;
            c1.PrimaryHost = uri.Host;
            c1.GetUrlsOfSite(sitePrimaryUrl);

            List <LocationUrls_Result> lstSItemapResult = new List <LocationUrls_Result>();

            foreach (string singleUrl in Crawl.Urls)
            {
                lstSItemapResult.Add(new LocationUrls_Result()
                {
                    Url = singleUrl, Changefreq = "weekly"
                });
            }
            //escludere ai e psd

            return(new XmlSitemapResult(lstSItemapResult));
        }