Exemplo n.º 1
0
        public static List <CrawlItem> LoadCSV(string csvDir, bool checkPrerender)
        {
            int count = 0;

            string[]         header = new string[4];
            List <CrawlItem> ret    = new List <CrawlItem>();

            using (var reader = new StreamReader(csvDir))
            {
                while (!reader.EndOfStream)
                {
                    var line   = reader.ReadLine();
                    var values = line.Split(',');
                    if (count == 0)
                    {
                    }
                    else if (count == 1)
                    {
                        header = values;
                    }
                    else
                    {
                        CrawlItem crawlItem = new CrawlItem
                        {
                            URL              = values[0],
                            PageTitle        = values[4],
                            MetaDescription1 = values[7],
                            MetaDescription2 = values[10],
                            MetaKeyword1     = values[16],
                            H1           = values[22],
                            H2           = values[26],
                            StatusCode   = values[3],
                            ResponseTime = values[45]
                        };
                        ret.Add(crawlItem);
                    }
                    count++;
                }
            }
            return(null);
        }
Exemplo n.º 2
0
        public static List <CrawlItem> LoadWithHelper(string path, bool fixPrerender)
        {
            int count = 0;

            List <CrawlItem> ret = new List <CrawlItem>();

            using (TextReader fileReader = File.OpenText(path))
            {
                var csv = new CsvReader(fileReader);
                csv.Configuration.HasHeaderRecord = false;
                while (csv.Read())
                {
                    CrawlItem crawlItem = null;

                    if (count != 1 && count != 0)
                    {
                        if (fixPrerender)
                        {
                            if (csv.GetField <string>(1) != "image/jpeg")
                            {
                                crawlItem = new CrawlItem
                                {
                                    URL              = csv.GetField <string>(0),
                                    PageTitle        = csv.GetField <string>(4),
                                    MetaDescription1 = csv.GetField <string>(7),
                                    MetaDescription2 = csv.GetField <string>(10),
                                    MetaKeyword1     = csv.GetField <string>(13),
                                    H1           = csv.GetField <string>(17),
                                    H2           = csv.GetField <string>(19),
                                    StatusCode   = csv.GetField <string>(3),
                                    ResponseTime = csv.GetField <string>(40)
                                };
                                crawlItem.RealURL = crawlItem.FixURL();
                            }
                        }
                        else
                        {
                            if (csv.GetField <string>(1) != "image/jpeg")
                            {
                                crawlItem = new CrawlItem
                                {
                                    URL              = csv.GetField <string>(0),
                                    PageTitle        = csv.GetField <string>(4),
                                    MetaDescription1 = csv.GetField <string>(7),
                                    MetaDescription2 = csv.GetField <string>(10),
                                    MetaKeyword1     = csv.GetField <string>(16),
                                    H1           = csv.GetField <string>(22),
                                    H2           = csv.GetField <string>(26),
                                    StatusCode   = csv.GetField <string>(3),
                                    ResponseTime = csv.GetField <string>(45)
                                };
                                crawlItem.RealURL = crawlItem.URL;
                            }
                        }
                        //if(control && fixPrerender){
                        //    crawlItem.RealURL = crawlItem.FixURL();
                        //}else{
                        //    crawlItem.RealURL = crawlItem.URL;
                        //}
                        if (crawlItem != null)
                        {
                            ret.Add(crawlItem);
                        }
                    }
                    count++;
                }
            }
            return(ret);
        }